From f904059b04119eadad905d6a1a8b911375b1a8b3 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:48:48 +0200 Subject: [PATCH 01/61] Add __init__.py to transformers directory Add init file to mark transformers as a package --- transformers/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 transformers/__init__.py diff --git a/transformers/__init__.py b/transformers/__init__.py new file mode 100644 index 0000000..c80e515 --- /dev/null +++ b/transformers/__init__.py @@ -0,0 +1 @@ +# Mark as package From b65a92a5a591206ce8859f3b7fe140f0da56b931 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:50:19 +0200 Subject: [PATCH 02/61] Add init file for domains sub-package --- transformers/domains/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 transformers/domains/__init__.py diff --git a/transformers/domains/__init__.py b/transformers/domains/__init__.py new file mode 100644 index 0000000..2996f29 --- /dev/null +++ b/transformers/domains/__init__.py @@ -0,0 +1 @@ +# Sub-package for different logic domains (url, firewall, etc.) From 5c80a8bacdb8dfaab0b17e14101446ef2538eddb Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:51:23 +0200 Subject: [PATCH 03/61] Create __init__.py --- transformers/domains/url/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 transformers/domains/url/__init__.py diff --git a/transformers/domains/url/__init__.py b/transformers/domains/url/__init__.py new file mode 100644 index 0000000..69badd6 --- /dev/null +++ b/transformers/domains/url/__init__.py @@ -0,0 +1 @@ +# url domain package From 8ae9c361e37e7296fb3fb7a66fcf39232ffce3e2 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:53:12 +0200 Subject: [PATCH 04/61] Define URL Domain Models with Pydantic This module defines the canonical schema for URLs, URL collections, and categories within the URL domain, abstracting vendor-specific constructs into a standardized representation for consistent policy handling. --- transformers/domains/url/models.py | 76 +++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/transformers/domains/url/models.py b/transformers/domains/url/models.py index 365a758..0d50bab 100644 --- a/transformers/domains/url/models.py +++ b/transformers/domains/url/models.py @@ -1,24 +1,66 @@ """ -models base class. +URL Domain Models - Unified Data Model (UDM) -Purpose: - +This module defines the canonical schema for URLs, URL collections, and +categories within the URL domain. It abstracts vendor-specific constructs +into a standardized representation to enable consistent policy handling +across heterogeneous security platforms. [cite: 51-52, 170] -Context: - Part of the use case within the Unified Policy Transformation Framework. +Design Principles: +- Domain-Level Logic: Operates purely on domain concepts. [cite: 99] +- Vendor-Agnostic: No vendor-specific logic is contained here. [cite: 100] +- Strong Typing: Enforces RFC-compliant formatting and normalization. [cite: 173] +""" -Responsibilities: - - - - - - +from typing import List, Literal, Optional +from datetime import datetime +from pydantic import BaseModel, Field, ConfigDict -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. -Author: - +class Category(BaseModel): + """ + Represents a normalized category entity with a stable identifier + and taxonomic classification. + """ + id: str = Field(..., description="Internal unique identifier for the category") + name: str = Field(..., description="Human-readable name of the category") + type: Literal["standard", "custom"] = Field( + ..., + description="Distinguishes between system-standard and user-defined categories " + ) -Created: - -""" + +class Metadata(BaseModel): + """ + Extensible container for enrichment data, such as timestamps and source info. + """ + processed_at: datetime = Field(..., description="Timestamp of when the record was processed") + source: Optional[str] = Field(None, description="The origin system of the data") + additional_info: Optional[dict] = Field(None, description="Placeholder for custom metadata expansion") + + +class URL_UDM(BaseModel): + """ + The Unified Data Model for URL entities. + + This model serves as the source of truth for processing, independent + of any external vendor system. + """ + # Performance optimization for Pydantic v2 + model_config = ConfigDict(populate_by_name=True) + + pattern: str = Field(..., description="The URL pattern (literal, wildcard, or regex) ") + type: Literal["literal", "wildcard", "regex"] = Field(..., description="The syntax type of the pattern") + action: Literal["allow", "block", "monitor"] = Field(..., description="Standardized enforcement action ") + status: Literal["enable", "disable"] = Field(..., description="Operational status of the rule") + url_list_id: str = Field(..., description="Unique ID for the parent URL list ") + url_list_name: str = Field(..., description="Human-readable name of the URL list") + + categories: List[Category] = Field( + default_factory=list, + description="Merged array of standard and custom categories " + ) + + vendor: Optional[str] = Field(None, description="Original vendor for traceability purposes ]") + metadata: Optional[Metadata] = Field(None, description="Processing metadata and timestamps") + notes: Optional[str] = Field(None, description="Optional justifications or comments ") From 6b3aef7a773ba89a23826b225e1638c20a0efbcf Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:54:32 +0200 Subject: [PATCH 05/61] Delete transformers/domains/url/transformers.py --- transformers/domains/url/transformers.py | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 transformers/domains/url/transformers.py diff --git a/transformers/domains/url/transformers.py b/transformers/domains/url/transformers.py deleted file mode 100644 index 0c16c5d..0000000 --- a/transformers/domains/url/transformers.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -transformers base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" From 68dea563474d2c9a66d272aaac787b09a99347ea Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:54:40 +0200 Subject: [PATCH 06/61] Delete transformers/domains/url/validators.py --- transformers/domains/url/validators.py | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 transformers/domains/url/validators.py diff --git a/transformers/domains/url/validators.py b/transformers/domains/url/validators.py deleted file mode 100644 index bf6b1e1..0000000 --- a/transformers/domains/url/validators.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -validators base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" From e76f0d17545394a47598668d0569097b9a02683d Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:56:17 +0200 Subject: [PATCH 07/61] Initialize vendors module with fortinet import --- transformers/domains/url/vendors/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 transformers/domains/url/vendors/__init__.py diff --git a/transformers/domains/url/vendors/__init__.py b/transformers/domains/url/vendors/__init__.py new file mode 100644 index 0000000..53f13d3 --- /dev/null +++ b/transformers/domains/url/vendors/__init__.py @@ -0,0 +1,4 @@ +from . import fortinet + +# Optional: List of vendors for easy iteration in orchestration logic +__all__ = ["fortinet"] From 2982fbec94bf18eb43800148f322825d96dece21 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:57:14 +0200 Subject: [PATCH 08/61] Update fortinet.py --- transformers/domains/url/vendors/fortinet.py | 221 +++++++++++++++++-- 1 file changed, 205 insertions(+), 16 deletions(-) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index 429e429..e3f6fc1 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -1,24 +1,213 @@ """ -fortinet transformers. +Fortinet URL Domain Integration -Purpose: - +This module implements the Transformer, Mapper, and Exporter for Fortinet, +converting between Fortinet-specific configurations and the Pydantic +Unified Data Model (UDM). +""" + +import jmespath +from datetime import datetime +from typing import Any, Dict, List, Optional + +# Framework imports - Absolute paths +from transformers.framework.udm_transformers.action_mapper import ActionMapper +from transformers.framework.udm_transformers.category_mapper import CategoryMapper +from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher +from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer +from transformers.framework.udm_transformers.type_mapper import TypeMapper + +# Domain Model imports +from transformers.domains.url.models import URL_UDM +from transformers.domains.url.models import Category +from transformers.domains.url.models import Metadata + +# ---------------- FORTINET MAPPINGS ---------------- -Context: - Part of the use case within the Unified Policy Transformation Framework. +FORTINET_ACTION_MAP = { + "allow": "allow", + "block": "block", + "monitor": "monitor", + "exempt": "allow" +} -Responsibilities: - - - - - - +FORTINET_CATEGORY_MAP = { + "3": "malware", + "4": "phishing", + "5": "gambling", + "default": "uncategorized", +} -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. +FORTINET_TYPE_MAP = { + "simple": "literal", + "wildcard": "wildcard", + "regex": "regex", +} -Author: - +# ---------------- EXTRACTION LAYER ---------------- -Created: - + +JMESPATH_FLATTEN_URLS = """ +*.urls.*.{ + pattern: url, + action: action, + status: status, + type: type, + url_id: url_id +} """ + +def flatten_fortinet_jmespath(raw_data): + flat = [] + + for _, url_list in raw_data.items(): + list_id = url_list["object_id"] + list_name = url_list["filter_name"] + + for _, item in url_list["urls"].items(): + flat.append({ + "pattern": item["url"], + "action": item["action"], + "status": item["status"], + "type": item["type"], + "url_id": item["url_id"], + "list_id": list_id, + "list_name": list_name, + "category_id": "Uncategorized" + }) + + return flat + +# ---------------- MAPPER & EXPORTER ---------------- + +class FortinetMapper: + """ + Handles semantic alignment and Pydantic UDM instantiation. + """ + + def to_udm(self, item: Dict[str, Any]) -> URL_UDM: + """ + Converts a transformed dictionary into a validated URL_UDM instance. + """ + # Map categories to the Category model + cat_id = item.get("category_id", "uncategorized") + categories = [ + Category(id=cat_id, name=cat_id.capitalize(), type="standard") + ] + + # Construct the Metadata model + # MetadataEnricher provides the ISO timestamp string + meta = Metadata( + processed_at=datetime.fromisoformat(item["metadata"]["processed_at"]), + ) + + return URL_UDM( + pattern=item["pattern"], + type=item["type"], + action=item["action"], + status="enable", + url_list_id=str(item["list_id"]), + url_list_name=item["list_name"], + categories=categories, + vendor=item["vendor"], + metadata=meta, + notes=item.get("notes") + ) + +class FortinetExporter: + """ + Universal Model -> Fortinet Format. + """ + def transform(self, udm: URL_UDM) -> Dict[str, Any]: + """ + Reconstructs Fortinet-specific pattern and type syntax. + """ + # Reverse mapping for Type [cite: 347] + reverse_type_map = {v: k for k, v in FORTINET_TYPE_MAP.items()} + + return { + "url": udm.pattern, + "type": reverse_type_map.get(udm.type, "simple") + } + +def run_universal_to_fortinet_pipeline(records: List[URL_UDM]) -> List[dict]: + output = [] + + for r in records: + output.append({ + "pattern": r.pattern, + "type": r.type, + "action": r.action, + "list_id": r.url_list_id, + "list_name": r.url_list_name + }) + + return output + + +def export_fortinet_json(records: List[dict]) -> dict: + grouped = {} + counters = {} + + for r in records: + key = r["list_id"] + + if key not in grouped: + grouped[key] = { + "object_id": r["list_id"], + "filter_name": r["list_name"], + "urls": {} + } + counters[key] = 0 + + idx = str(counters[key]) + counters[key] += 1 + + grouped[key]["urls"][idx] = { + "url_id": str(counters[key]), + "url": r["pattern"], + "type": r["type"], + "action": r["action"], + "status": "enable" + } + + return grouped + +# ---------------- EXECUTION PIPELINE ---------------- + +def run_fortinet_to_universal_pipeline(raw_data: Dict[str, Any]) -> List[URL_UDM]: + """ + Orchestrates the deterministic flow from raw Fortinet data to UDM objects. + """ + # 1. Extraction + flat_data = flatten_fortinet_jmespath(raw_data) + + # 2. Transformation Pipeline + steps = [ + ActionMapper(FORTINET_ACTION_MAP), + PatternNormalizer(), + TypeMapper(FORTINET_TYPE_MAP), + CategoryMapper(FORTINET_CATEGORY_MAP), + MetadataEnricher("fortinet") + ] + + mapper = FortinetMapper() + udm_records = [] + + for record in flat_data: + # Apply each modular transformation unit + for step in steps: + record = step.transform(record) + + # 3. Validation & Pydantic Conversion + udm_records.append(mapper.to_udm(record)) + + return udm_records + +# ---------------- REGISTRATION ---------------- + +# This is what debugPythonScript.py is looking for +VENDOR_TO_UNIVERSAL_PIPELINES = { + "fortinet": run_fortinet_to_universal_pipeline +} + From 1bc4050d289b03f5b6ea0a9c0b2316e7d8f05b26 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:58:11 +0200 Subject: [PATCH 09/61] Update netskope.py --- transformers/domains/url/vendors/netskope.py | 207 +++++++++++++++++-- 1 file changed, 191 insertions(+), 16 deletions(-) diff --git a/transformers/domains/url/vendors/netskope.py b/transformers/domains/url/vendors/netskope.py index 5e54d71..d106cfc 100644 --- a/transformers/domains/url/vendors/netskope.py +++ b/transformers/domains/url/vendors/netskope.py @@ -1,24 +1,199 @@ """ -netskope transformers. +Netskope URL Domain Integration -Purpose: - +This module implements the Transformer, Mapper, and Exporter for Netskope, +converting between Netskope-specific configurations and the Pydantic +Unified Data Model (UDM). +""" + +import re +import jmespath +from datetime import datetime +from typing import Any, Dict, List, Optional + +# Framework imports - Absolute paths +from transformers.framework.udm_transformers.action_mapper import ActionMapper +from transformers.framework.udm_transformers.category_mapper import CategoryMapper +from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher +from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer +from transformers.framework.udm_transformers.type_mapper import TypeMapper +from transformers.framework.udm_transformers.base_transformer import BaseTransformer + +# Domain Model imports +from transformers.domains.url.models import URL_UDM +from transformers.domains.url.models import Category +from transformers.domains.url.models import Metadata -Context: - Part of the use case within the Unified Policy Transformation Framework. +# ---------------- NETSKOPE MAPPINGS ---------------- -Responsibilities: - - - - - - +NETSKOPE_ACTION_MAP = { + "block": "deny", + "allow": "allow", + "monitor": "allow", +} -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. +NETSKOPE_CATEGORY_MAP = { + "malware": "malware", + "phishing": "phishing", + "gambling": "gambling", + "uncategorized": "uncategorized", +} -Author: - +NETSKOPE_TO_UNIVERSAL_TYPE_MAP = { + "exact": "literal", + "regex": "regex", +} -Created: - +# ---------------- EXTRACTION LAYER ---------------- + +JMESPATH_NETSKOPE = """ +values(@)[?modify_type!='Deleted'].{ + list_name: name, + list_id: object_id, + type: data_type, + urls: values(data_urls) +} """ + +def flatten_netskope_jmespath(url_lists: Dict[str, Any]) -> List[Dict[str, Any]]: + """Flatten the structure using jmespath. """ + extracted = jmespath.search(JMESPATH_NETSKOPE, url_lists) or [] + flat = [] + + for lst in extracted: + for entry in lst.get("urls", []): + url = entry.get("url") + if not url: + continue + flat.append({ + "pattern": url, + "action": "allow", + "category_id": "Uncategorized", + "list_name": lst["list_name"], + "list_id": str(lst["list_id"]), + "type": lst["type"] + }) + return flat + +# ---------------- TRANSFORMERS ---------------- + +class NetskopePatternNormalizer(BaseTransformer): + """Normalize Netskope URL patterns for vendor compatibility.""" + + def wildcard_to_regex(self, pattern: str) -> str: + """Convert a wildcard domain pattern to a regex.""" + if pattern.startswith("*."): + domain = pattern[2:].replace(".", r"\.") + return rf"^([^.]+\.)*{domain}$" + return pattern + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + raw_pattern = item.get("pattern", "") + universal_type = item.get("type", "literal") + + if universal_type in ("literal", "exact"): + item["pattern"] = raw_pattern + item["netskope_type"] = "exact" + elif universal_type in ("wildcard", "regex"): + item["pattern"] = self.wildcard_to_regex(raw_pattern) + item["netskope_type"] = "regex" + else: + item["pattern"] = raw_pattern + item["netskope_type"] = "exact" + return item + +class NetskopePatternDenormalizer(BaseTransformer): + """Convert Netskope patterns back to universal model patterns.""" + + def regex_to_wildcard(self, pattern: str) -> Optional[str]: + wildcard_regex = r"^\^\(\[\^\.\]\+\\\.\)\*(.+)\\\.([a-zA-Z0-9\-]+)\$$" + match = re.match(wildcard_regex, pattern) + if match: + domain = f"{match.group(1)}.{match.group(2)}" + return f"*.{domain}" + return None + + def is_regex(self, pattern: str) -> bool: + regex_markers = ("^", "$", "(", ")", "[", "]", "+", "?", "|", "{", "}") + return any(marker in pattern for marker in regex_markers) + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + pattern = item.get("pattern", "").replace("\\\\", "\\") + if pattern.startswith("*.") and pattern.count("*") == 1: + item["type"] = "wildcard" + elif "*" in pattern: + item["type"] = "regex" + elif self.is_regex(pattern): + wildcard = self.regex_to_wildcard(pattern) + if wildcard: + item["type"] = "wildcard" + pattern = wildcard + else: + item["type"] = "regex" + else: + item["type"] = "exact" + item["pattern"] = pattern + item.pop("netskope_type", None) + return item + +# ---------------- MAPPER & EXPORTER ---------------- + +class NetskopeMapper: + """Handles semantic alignment and Pydantic UDM instantiation.""" + + def to_udm(self, item: Dict[str, Any]) -> URL_UDM: + """Converts transformed dictionary into validated URL_UDM instance. """ + cat_id = item.get("category_id", "uncategorized") + categories = [Category(id=cat_id, name=cat_id.capitalize(), type="standard")] + + meta = Metadata( + processed_at=datetime.fromisoformat(item["metadata"]["processed_at"]), + source="netskope" + ) + + return URL_UDM( + pattern=item["pattern"], + type=item["type"], + action=item["action"], + status="enable", + url_list_id=item["list_id"], + url_list_name=item["list_name"], + categories=categories, + vendor=item["vendor"], + metadata=meta, + notes=item.get("notes") + ) + +class NetskopeExporter: + """Universal Model -> Netskope Format. """ + + def transform(self, udm: URL_UDM) -> Dict[str, Any]: + # Implementation would use NetskopePatternNormalizer logic here + pass + +# ---------------- EXECUTION PIPELINE ---------------- + +def run_netskope_to_universal_pipeline(raw_data: Dict[str, Any]) -> List[URL_UDM]: + """Orchestrates the flow from raw Netskope data to UDM objects. """ + # 1. Extraction + flat_data = flatten_netskope_jmespath(raw_data) + + # 2. Transformation Pipeline + steps = [ + ActionMapper(NETSKOPE_ACTION_MAP), + TypeMapper(NETSKOPE_TO_UNIVERSAL_TYPE_MAP), + NetskopePatternDenormalizer(), + CategoryMapper(NETSKOPE_CATEGORY_MAP), + MetadataEnricher("netskope") + ] + + mapper = NetskopeMapper() + udm_records = [] + + for record in flat_data: + for step in steps: + record = step.transform(record) + udm_records.append(mapper.to_udm(record)) + + return udm_records + From 7971c444499c99df57a19194ac3245ba4983304f Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:59:21 +0200 Subject: [PATCH 10/61] Add package declaration to __init__.py --- transformers/framework/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 transformers/framework/__init__.py diff --git a/transformers/framework/__init__.py b/transformers/framework/__init__.py new file mode 100644 index 0000000..5bb534f --- /dev/null +++ b/transformers/framework/__init__.py @@ -0,0 +1 @@ +# package From 3cda88319ad04888bcc11bdbf9f08335a29948b3 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 14:59:54 +0200 Subject: [PATCH 11/61] Delete transformers/framework/pipeline.py --- transformers/framework/pipeline.py | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 transformers/framework/pipeline.py diff --git a/transformers/framework/pipeline.py b/transformers/framework/pipeline.py deleted file mode 100644 index e6137c7..0000000 --- a/transformers/framework/pipeline.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -pipeline base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" From e490f4c7e8ed7007a790b40b264260bf1d8e11da Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:00:18 +0200 Subject: [PATCH 12/61] Create pipelines.py --- transformers/framework/pipelines.py | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 transformers/framework/pipelines.py diff --git a/transformers/framework/pipelines.py b/transformers/framework/pipelines.py new file mode 100644 index 0000000..6967ea8 --- /dev/null +++ b/transformers/framework/pipelines.py @@ -0,0 +1,39 @@ +"""Pipeline helper functions for URL transformations. + +This module provides utilities to apply a sequence of transformers +to vendor configuration items, producing universal model dictionaries. +""" + +from typing import Any +from typing import Dict +from typing import List + +from transformers.framework.udm_transformers.base_transformer import BaseTransformer + + +def apply_transformers( + items: List[Dict[str, Any]], + transformers: List[BaseTransformer] +) -> List[Dict[str, Any]]: + """Apply a sequence of transformers to a list of items. + + Each item in the input list is processed sequentially by all + transformers in the given order. + + Args: + items: A list of dictionaries representing vendor configuration + entries. + transformers: An ordered list of transformer instances that + implement the `transform` method. + + Returns: + A list of transformed dictionaries. + """ + result: List[Dict[str, Any]] = [] + + for item in items: + for transformer in transformers: + item = transformer.transform(item) + result.append(item) + + return result From f1bc3eb648106ee7bbad2b60cb8a216ddbb02944 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:00:38 +0200 Subject: [PATCH 13/61] Delete transformers/framework/context.py --- transformers/framework/context.py | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 transformers/framework/context.py diff --git a/transformers/framework/context.py b/transformers/framework/context.py deleted file mode 100644 index fd6081f..0000000 --- a/transformers/framework/context.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -context base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" From aa9b34be45005255bffeac2b6c1ddef28e396602 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:01:02 +0200 Subject: [PATCH 14/61] Delete transformers/framework/transformers directory --- .../transformers/generic_transformers.py | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100644 transformers/framework/transformers/generic_transformers.py diff --git a/transformers/framework/transformers/generic_transformers.py b/transformers/framework/transformers/generic_transformers.py deleted file mode 100644 index 9396226..0000000 --- a/transformers/framework/transformers/generic_transformers.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -generic_transformers base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" From bef8b82807c8c31865657655da1787f27b9d532a Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:01:53 +0200 Subject: [PATCH 15/61] Create __init__.py --- transformers/framework/udm_transformers/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 transformers/framework/udm_transformers/__init__.py diff --git a/transformers/framework/udm_transformers/__init__.py b/transformers/framework/udm_transformers/__init__.py new file mode 100644 index 0000000..5bb534f --- /dev/null +++ b/transformers/framework/udm_transformers/__init__.py @@ -0,0 +1 @@ +# package From 4898729ce1c7d33395e3f93aacf5fa5446a7b708 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:02:30 +0200 Subject: [PATCH 16/61] Create action_mapper.py --- .../udm_transformers/action_mapper.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 transformers/framework/udm_transformers/action_mapper.py diff --git a/transformers/framework/udm_transformers/action_mapper.py b/transformers/framework/udm_transformers/action_mapper.py new file mode 100644 index 0000000..111b5a6 --- /dev/null +++ b/transformers/framework/udm_transformers/action_mapper.py @@ -0,0 +1,46 @@ +"""Action mapping transformer. + +This module defines a transformer responsible for mapping action values +between vendor-specific representations and the universal data model. +""" + +from typing import Any +from typing import Dict + +# Change from: from .base_transformer import BaseTransformer +# To the absolute framework path: +from transformers.framework.udm_transformers.base_transformer import BaseTransformer + +class ActionMapper(BaseTransformer): + """Map action values between vendor and universal models. + + This transformer replaces the ``action`` field of an item using a + predefined mapping dictionary. If the action is not found in the + mapping, it is left unchanged. + """ + + def __init__(self, action_map: Dict[str, str]) -> None: + """Initialize the ActionMapper. + + Args: + action_map: A dictionary mapping source action values to + destination action values. + """ + self.action_map = action_map + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Transform an item's action field using the action mapping. + + Args: + item: A dictionary representing a single rule or configuration + entry containing an ``action`` field. + + Returns: + The transformed item with its ``action`` field mapped according + to the configured action map. + """ + action = item.get("action") + if action in self.action_map: + item["action"] = self.action_map[action] + + return item From 62dd50c5a6e6f25a12ba68af9bfe94e070c8970a Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:03:04 +0200 Subject: [PATCH 17/61] Create BaseTransformer abstract class for transformers This module defines the abstract base class for all transformers, ensuring a consistent interface for reusable transformation components. --- .../udm_transformers/base_transformer.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 transformers/framework/udm_transformers/base_transformer.py diff --git a/transformers/framework/udm_transformers/base_transformer.py b/transformers/framework/udm_transformers/base_transformer.py new file mode 100644 index 0000000..e689d2c --- /dev/null +++ b/transformers/framework/udm_transformers/base_transformer.py @@ -0,0 +1,36 @@ +""" +Base Transformer Definition - Framework Layer + +This module defines the abstract base class (ABC) used by all transformers +within the generic transformation engine. It ensures a consistent interface +for reusable transformation components. +""" + +from abc import ABC +from abc import abstractmethod +from typing import Any +from typing import Dict + + +class BaseTransformer(ABC): + """ + Define the interface for all transformers. + + All concrete transformers must implement the ``transform`` method, + which takes a single dictionary item and returns a transformed + dictionary. This maintains a domain-agnostic execution model. + """ + + @abstractmethod + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """ + Transform a single dictionary item. + + Args: + item: A dictionary representing a single configuration or + URL entry. + + Returns: + A transformed dictionary. + """ + raise NotImplementedError From 7fb33c0569771666609d14561a5f77200354e9fd Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:03:38 +0200 Subject: [PATCH 18/61] Add CategoryMapper for category ID transformation This module defines a transformer responsible for mapping category identifiers between vendor-specific representations and the universal data model. --- .../udm_transformers/category_mapper.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 transformers/framework/udm_transformers/category_mapper.py diff --git a/transformers/framework/udm_transformers/category_mapper.py b/transformers/framework/udm_transformers/category_mapper.py new file mode 100644 index 0000000..c731a9c --- /dev/null +++ b/transformers/framework/udm_transformers/category_mapper.py @@ -0,0 +1,47 @@ +"""Category mapping transformer. + +This module defines a transformer responsible for mapping category +identifiers between vendor-specific representations and the universal +data model. +""" + +from typing import Any +from typing import Dict + +# Restructured to use the absolute path within the Framework layer +from transformers.framework.udm_transformers.base_transformer import BaseTransformer + + +class CategoryMapper(BaseTransformer): + """Map category identifiers between vendor and universal models. + + This transformer replaces the ``category_id`` field of an item using + a predefined mapping dictionary. If the category is not found in the + mapping, it is left unchanged. + """ + + def __init__(self, category_map: Dict[str, str]) -> None: + """Initialize the CategoryMapper. + + Args: + category_map: A dictionary mapping source category identifiers + to destination category identifiers. + """ + self.category_map = category_map + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Transform an item's category identifier using the category map. + + Args: + item: A dictionary representing a single rule or configuration + entry containing a ``category_id`` field. + + Returns: + The transformed item with its ``category_id`` field mapped + according to the configured category map. + """ + category_id = item.get("category_id") + if category_id in self.category_map: + item["category_id"] = self.category_map[category_id] + + return item From 18943ced17739550c9d8ec68d7472e16332acb8f Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:04:13 +0200 Subject: [PATCH 19/61] Create metadata_enricher.py --- .../udm_transformers/metadata_enricher.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 transformers/framework/udm_transformers/metadata_enricher.py diff --git a/transformers/framework/udm_transformers/metadata_enricher.py b/transformers/framework/udm_transformers/metadata_enricher.py new file mode 100644 index 0000000..6206e08 --- /dev/null +++ b/transformers/framework/udm_transformers/metadata_enricher.py @@ -0,0 +1,47 @@ +"""Metadata enrichment transformer. + +This module defines a transformer that adds vendor information and +metadata timestamps to each item in the transformation pipeline. +""" + +from datetime import datetime +from typing import Any +from typing import Dict + +# Restructured to use the absolute path within the Framework layer +from transformers.framework.udm_transformers.base_transformer import BaseTransformer + + +class MetadataEnricher(BaseTransformer): + """Enrich items with vendor and metadata information. + + This transformer adds a ``vendor`` field and a ``metadata`` dictionary + containing a ``processed_at`` timestamp to each item. + """ + + def __init__(self, vendor: str) -> None: + """Initialize the MetadataEnricher. + + Args: + vendor: The vendor name to attach to each item. + """ + self.vendor = vendor + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Add vendor and metadata information to an item. + + Args: + item: A dictionary representing a single configuration or URL entry. + + Returns: + The transformed dictionary containing the ``vendor`` field and + a ``metadata.processed_at`` timestamp. + """ + item["vendor"] = self.vendor + if "metadata" not in item: + item["metadata"] = {} + + # Standardizing to the UDM requirement for processed_at timestamps + item["metadata"]["processed_at"] = datetime.utcnow().isoformat() + + return item From 9878cde89ddbd52e23c5ce7eadcd1786904bee12 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:04:48 +0200 Subject: [PATCH 20/61] Implement pattern normalizer transformer This module defines a pattern normalizer transformer that ensures each item has a 'pattern' field, initializing it to an empty string if missing. --- .../udm_transformers/pattern_normalizer.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 transformers/framework/udm_transformers/pattern_normalizer.py diff --git a/transformers/framework/udm_transformers/pattern_normalizer.py b/transformers/framework/udm_transformers/pattern_normalizer.py new file mode 100644 index 0000000..ed315a5 --- /dev/null +++ b/transformers/framework/udm_transformers/pattern_normalizer.py @@ -0,0 +1,32 @@ +"""Pattern normalization transformer. + +This module defines a generic pattern normalizer that ensures each item +has a ``pattern`` field. Currently, this transformer acts as a pass-through +but serves as a hook for formal pattern representations. +""" + +from typing import Any +from typing import Dict + +# Restructured to use the absolute path within the Framework layer +from transformers.framework.udm_transformers.base_transformer import BaseTransformer + + +class PatternNormalizer(BaseTransformer): + """Normalize or enforce the presence of a pattern field in items. + + This transformer guarantees that each dictionary item contains a + ``pattern`` key. If the key is missing, it is initialized to an empty string. + """ + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Ensure the item has a pattern field. + + Args: + item: A dictionary representing a single configuration or URL entry. + + Returns: + The same dictionary with a ``pattern`` key ensured. + """ + item["pattern"] = item.get("pattern", "") + return item From 0293247c31b46b31890173a429c27104e2b48b8f Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:05:21 +0200 Subject: [PATCH 21/61] Add TypeMapper for type mapping transformations This module defines a transformer that maps item types between vendor-specific representations and the universal data model. It includes a method to transform an item's type field using a predefined mapping dictionary. --- .../framework/udm_transformers/type_mapper.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 transformers/framework/udm_transformers/type_mapper.py diff --git a/transformers/framework/udm_transformers/type_mapper.py b/transformers/framework/udm_transformers/type_mapper.py new file mode 100644 index 0000000..8285604 --- /dev/null +++ b/transformers/framework/udm_transformers/type_mapper.py @@ -0,0 +1,47 @@ +"""Type mapping transformer. + +This module defines a transformer that maps item types (e.g., literal, +wildcard, regex, substring) between vendor-specific representations +and the universal data model. +""" + +from typing import Any +from typing import Dict + +# Restructured to use the absolute path within the Framework layer +from transformers.framework.udm_transformers.base_transformer import BaseTransformer + + +class TypeMapper(BaseTransformer): + """Map type values between vendor and universal models. + + This transformer replaces the ``type`` field of an item using a + predefined mapping dictionary. If the type is not found in the + mapping, it is left unchanged. + """ + + def __init__(self, type_map: Dict[str, str]) -> None: + """Initialize the TypeMapper. + + Args: + type_map: A dictionary mapping source type values to + destination type values. + """ + self.type_map = type_map + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Transform an item's type field using the type mapping. + + Args: + item: A dictionary representing a single rule or configuration + entry containing a ``type`` field. + + Returns: + The transformed item with its ``type`` field mapped according + to the configured type map. + """ + item_type = item.get("type") + if item_type in self.type_map: + item["type"] = self.type_map[item_type] + + return item From c94fd360691eab429722abc87a8a73c571a41f7d Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:05:59 +0200 Subject: [PATCH 22/61] Delete transformers/type_mapper.py --- transformers/type_mapper.py | 46 ------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 transformers/type_mapper.py diff --git a/transformers/type_mapper.py b/transformers/type_mapper.py deleted file mode 100644 index 79a6afd..0000000 --- a/transformers/type_mapper.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Type mapping transformer. - -This module defines a transformer that maps item types (e.g., literal, -wildcard, regex, substring) between vendor-specific representations -and the universal data model. -""" - -from typing import Any -from typing import Dict - -from .base_transformer import BaseTransformer - - -class TypeMapper(BaseTransformer): - """Map type values between vendor and universal models. - - This transformer replaces the ``type`` field of an item using a - predefined mapping dictionary. If the type is not found in the - mapping, it is left unchanged. - """ - - def __init__(self, type_map: Dict[str, str]) -> None: - """Initialize the TypeMapper. - - Args: - type_map: A dictionary mapping source type values to - destination type values. - """ - self.type_map = type_map - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Transform an item's type field using the type mapping. - - Args: - item: A dictionary representing a single rule or configuration - entry containing a ``type`` field. - - Returns: - The transformed item with its ``type`` field mapped according - to the configured type map. - """ - item_type = item.get("type") - if item_type in self.type_map: - item["type"] = self.type_map[item_type] - - return item From 8d5d682b5dc59be5b1d5685a7f66cd5a726681e1 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:06:10 +0200 Subject: [PATCH 23/61] Delete transformers/transformers.py --- transformers/transformers.py | 371 ----------------------------------- 1 file changed, 371 deletions(-) delete mode 100644 transformers/transformers.py diff --git a/transformers/transformers.py b/transformers/transformers.py deleted file mode 100644 index 4f55395..0000000 --- a/transformers/transformers.py +++ /dev/null @@ -1,371 +0,0 @@ -""" -Single-file SDK for bidirectional URL-filter migration transformers. - -This module provides: -- Universal intermediate URL-filter model -- Vendor-specific transformers for: Fortinet, Netskope, Zscaler, Prisma Access -- Modular basic transformers (action, pattern, type, category, metadata) -- Transformer pipelines (vendor → universal and universal → vendor) -- CLI and unit-test support -- Built-in sample mappings for immediate use -""" - -from dataclasses import dataclass -from dataclasses import field -from typing import Any -from typing import Dict -from typing import List - -# ---------------- UNIVERSAL DATA MODEL ---------------- - - -@dataclass -class UniversalURLFilter: - """Universal URL Filter Model.""" - - pattern: str - action: str - category: str - list_name: str = "" - list_id: str = "" - type: str = "literal" - vendor: str = "" - metadata: Dict[str, Any] = field(default_factory=dict) - - -# ---------------- BASIC TRANSFORMERS ---------------- - - -class BaseTransformer: - """Abstract base class for all transformers.""" - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """ - Transform a single item. - - Args: - item: Input dictionary from vendor configuration. - - Returns: - Transformed dictionary. - - Raises: - NotImplementedError: If the subclass does not override this method. - """ - raise NotImplementedError - - -class ActionMapper(BaseTransformer): - """Map vendor actions to universal actions (or reverse).""" - - def __init__(self, action_map: Dict[str, str]): - """ - Initialize the ActionMapper. - - Args: - action_map: Mapping from vendor action → universal action. - """ - self.action_map = action_map - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """ - Map the action field using the configured mapping. - - Args: - item: Input dictionary. - - Returns: - Dictionary with updated action. - """ - item["action"] = self.action_map.get(item.get("action"), item.get("action")) - return item - - -class PatternNormalizer(BaseTransformer): - """Normalize or pass through patterns unchanged.""" - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """ - Normalize the pattern field. - - Args: - item: Input dictionary. - - Returns: - Updated dictionary with normalized pattern. - """ - item["pattern"] = item.get("pattern", "") - return item - - -class NetskopePatternNormalizer(BaseTransformer): - """Normalize Netskope patterns and convert wildcards to regex formats.""" - - def wildcard_to_regex(self, pattern: str) -> str: - """ - Convert a Netskope wildcard pattern into a regex expression. - - Args: - pattern: Input wildcard pattern. - - Returns: - Regex-safe version of the pattern. - """ - if pattern.startswith("*."): - domain = pattern[2:] - domain = domain.replace(".", r"\.") - return rf"^([^.]+\.)*{domain}$" - else: - escaped = pattern.replace(".", r"\.") - return rf"^{escaped}$" - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """ - Transform Netskope patterns into the correct regex or literal format. - - Args: - item: Input dictionary. - - Returns: - Updated dictionary with Netskope-normalized pattern. - """ - raw = item.get("pattern", "") - utype = item.get("type", "literal") - - # literal/exact - if utype in ("literal", "exact"): - final = raw - item["netskope_type"] = "exact" - - # wildcard - elif utype == "wildcard": - final = self.wildcard_to_regex(raw) - item["netskope_type"] = "regex" - - # regex - elif utype == "regex": - final = raw - item["netskope_type"] = "regex" - - # substring → literal - else: - final = raw - item["netskope_type"] = "exact" - - # escape ONCE for JSON - final = final.replace("\\", "\\\\") - item["pattern"] = final - - return item - - -class TypeMapper(BaseTransformer): - """Map vendor pattern types to universal types (or reverse).""" - - def __init__(self, type_map: Dict[str, str]): - """ - Initialize the TypeMapper. - - Args: - type_map: Mapping from vendor type → universal type. - """ - self.type_map = type_map - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """ - Map the pattern type field. - - Vendor types may be inconsistent (uppercase/lowercase), - so normalization rules apply before mapping. - - Args: - item: Input dictionary. - - Returns: - Updated dictionary with normalized pattern type. - """ - vendor_type = item.get("type", "simple") - - vendor_type_normalized = ( - vendor_type.upper() - if vendor_type in ["STRING", "WILDCARD", "REGEX"] - else vendor_type.lower() - ) - - item["type"] = self.type_map.get(vendor_type_normalized, "literal") - return item - - -class CategoryMapper(BaseTransformer): - """Map vendor category IDs or names to universal names.""" - - def __init__(self, category_map: Dict[str, str]): - """ - Initialize the CategoryMapper. - - Args: - category_map: Mapping from vendor category → universal. - """ - self.category_map = category_map - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """ - Map the category field. - - Args: - item: Input dictionary. - - Returns: - Updated dictionary with normalized category. - """ - vendor_cat = str(item.get("category_id", "default")) - item["category"] = self.category_map.get(vendor_cat, "uncategorized") - return item - - -class MetadataEnricher(BaseTransformer): - """Enrich items with vendor name and embedded metadata fields.""" - - def __init__(self, vendor: str, extra_fields: List[str] | None = None): - """ - Initialize the MetadataEnricher. - - Args: - vendor: Vendor identifier. - extra_fields: Optional list of fields to copy to metadata. - """ - self.vendor = vendor - self.extra_fields = extra_fields or [] - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """ - Add vendor metadata fields to the item. - - Args: - item: Input dictionary. - - Returns: - Updated dictionary enriched with metadata. - """ - item["vendor"] = self.vendor - item.setdefault("metadata", {}) - - for field_name in self.extra_fields: - if field_name in item: - item["metadata"][field_name] = item[field_name] - - return item - - -# ---------------- PIPELINE EXECUTOR ---------------- - - -def apply_transformers( - items: List[Dict[str, Any]], transformers: List[BaseTransformer] -) -> List[Dict[str, Any]]: - """ - Apply a list of transformers sequentially to a list of items. - - Args: - items: List of vendor configuration dictionaries. - transformers: Ordered list of transformer instances. - - Returns: - List of transformed dictionaries. - """ - result = [] - for item in items: - for transformer in transformers: - item = transformer.transform(item) - result.append(item) - return result - - -# ---------------- VENDOR MAPPINGS ---------------- - -FORTINET_ACTION_MAP = {"block": "block", "allow": "allow", "monitor": "monitor"} -FORTINET_CATEGORY_MAP = {"3": "malware", "4": "phishing", "5": "gambling", "default": "uncategorized"} -FORTINET_TYPE_MAP = {"simple": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} - -# ---------------- NETSKOPE ---------------- - -NETSKOPE_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "monitor"} -NETSKOPE_CATEGORY_MAP = {"malware": "malware", "phishing": "phishing", "gambling": "gambling", "uncategorized": "uncategorized"} -NETSKOPE_TO_UNIVERSAL_TYPE_MAP = {"exact": "literal", "regex": "regex"} -UNIVERSAL_TO_NETSKOPE_TYPE_MAP = {"literal": "exact", "regex": "regex", "wildcard": "regex", "substring": "regex"} - -# ---------------- ZSCALER ---------------- - -ZSCALER_ACTION_MAP = {"block": "BLOCK", "allow": "ALLOW", "monitor": "MONITOR"} -ZSCALER_CATEGORY_MAP = {"malware": "malware", "phishing": "phishing", "gambling": "gambling", "uncategorized": "uncategorized"} -ZSCALER_TYPE_MAP = {"STRING": "literal", "WILDCARD": "wildcard", "REGEX": "regex"} - -# ---------------- PALO ALTO ---------------- - -PRISMA_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "alert"} -PRISMA_CATEGORY_MAP = {"malware": "malware", "phishing": "phishing", "gambling": "gambling", "uncategorized": "uncategorized"} -PRISMA_TYPE_MAP = {"simple": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} - -# ---------------- PIPELINE DEFINITIONS ---------------- - -VENDOR_TO_UNIVERSAL_PIPELINES = { - "fortinet": [ - ActionMapper(FORTINET_ACTION_MAP), - PatternNormalizer(), - TypeMapper(FORTINET_TYPE_MAP), - CategoryMapper(FORTINET_CATEGORY_MAP), - MetadataEnricher("fortinet"), - ], - "netskope": [ - ActionMapper(NETSKOPE_ACTION_MAP), - PatternNormalizer(), - TypeMapper(NETSKOPE_TO_UNIVERSAL_TYPE_MAP), - CategoryMapper(NETSKOPE_CATEGORY_MAP), - MetadataEnricher("netskope"), - ], - "zscaler": [ - ActionMapper(ZSCALER_ACTION_MAP), - PatternNormalizer(), - TypeMapper(ZSCALER_TYPE_MAP), - CategoryMapper(ZSCALER_CATEGORY_MAP), - MetadataEnricher("zscaler"), - ], - "prisma": [ - ActionMapper(PRISMA_ACTION_MAP), - PatternNormalizer(), - TypeMapper(PRISMA_TYPE_MAP), - CategoryMapper(PRISMA_CATEGORY_MAP), - MetadataEnricher("prisma"), - ], -} - -UNIVERSAL_TO_VENDOR_PIPELINES = { - "fortinet": [ - ActionMapper({v: k for k, v in FORTINET_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({v: k for k, v in FORTINET_TYPE_MAP.items()}), - CategoryMapper({v: k for k, v in FORTINET_CATEGORY_MAP.items()}), - MetadataEnricher("fortinet"), - ], - "netskope": [ - ActionMapper({v: k for k, v in NETSKOPE_ACTION_MAP.items()}), - NetskopePatternNormalizer(), - TypeMapper(UNIVERSAL_TO_NETSKOPE_TYPE_MAP), - CategoryMapper({v: k for k, v in NETSKOPE_CATEGORY_MAP.items()}), - MetadataEnricher("netskope"), - ], - "zscaler": [ - ActionMapper({v: k for k, v in ZSCALER_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({v: k for k, v in ZSCALER_TYPE_MAP.items()}), - CategoryMapper({v: k for k, v in ZSCALER_CATEGORY_MAP.items()}), - MetadataEnricher("zscaler"), - ], - "prisma": [ - ActionMapper({v: k for k, v in PRISMA_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({v: k for k, v in PRISMA_TYPE_MAP.items()}), - CategoryMapper({v: k for k, v in PRISMA_CATEGORY_MAP.items()}), - MetadataEnricher("prisma"), - ], -} From 4ab9ca767d0702d675716b7a1b394646e8530f6d Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:06:31 +0200 Subject: [PATCH 24/61] Delete transformers/pipelines.py --- transformers/pipelines.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 transformers/pipelines.py diff --git a/transformers/pipelines.py b/transformers/pipelines.py deleted file mode 100644 index 1ec2647..0000000 --- a/transformers/pipelines.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Pipeline helper functions for URL transformations. - -This module provides utilities to apply a sequence of transformers -to vendor configuration items, producing universal model dictionaries. -""" - -from typing import Any -from typing import Dict -from typing import List - -from transformers.base_transformer import BaseTransformer - - -def apply_transformers( - items: List[Dict[str, Any]], - transformers: List[BaseTransformer] -) -> List[Dict[str, Any]]: - """Apply a sequence of transformers to a list of items. - - Each item in the input list is processed sequentially by all - transformers in the given order. - - Args: - items: A list of dictionaries representing vendor configuration - entries. - transformers: An ordered list of transformer instances that - implement the `transform` method. - - Returns: - A list of transformed dictionaries. - """ - result: List[Dict[str, Any]] = [] - - for item in items: - for transformer in transformers: - item = transformer.transform(item) - result.append(item) - - return result From 668ab1d865351184dde30e1bf7c6d692f5a06f07 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:06:40 +0200 Subject: [PATCH 25/61] Delete transformers/pattern_normalizer.py --- transformers/pattern_normalizer.py | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 transformers/pattern_normalizer.py diff --git a/transformers/pattern_normalizer.py b/transformers/pattern_normalizer.py deleted file mode 100644 index f88212d..0000000 --- a/transformers/pattern_normalizer.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Pattern normalization transformer. - -This module defines a generic pattern normalizer that ensures each item -has a ``pattern`` field. Currently, this transformer acts as a pass-through. -""" - -from typing import Any -from typing import Dict - -from .base_transformer import BaseTransformer - - -class PatternNormalizer(BaseTransformer): - """Normalize or enforce the presence of a pattern field in items. - - This transformer guarantees that each dictionary item contains a - ``pattern`` key. If the key is missing, it is initialized to an empty string. - """ - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Ensure the item has a pattern field. - - Args: - item: A dictionary representing a single configuration or URL entry. - - Returns: - The same dictionary with a ``pattern`` key ensured. - """ - item["pattern"] = item.get("pattern", "") - return item From 6bd626fe240ff08a3f3eaf2c84924bab378dddd8 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:06:49 +0200 Subject: [PATCH 26/61] Delete transformers/metadata_enricher.py --- transformers/metadata_enricher.py | 45 ------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 transformers/metadata_enricher.py diff --git a/transformers/metadata_enricher.py b/transformers/metadata_enricher.py deleted file mode 100644 index 9566977..0000000 --- a/transformers/metadata_enricher.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Metadata enrichment transformer. - -This module defines a transformer that adds vendor information and -metadata timestamps to each item in the transformation pipeline. -""" - -from datetime import datetime -from typing import Any -from typing import Dict - -from .base_transformer import BaseTransformer - - -class MetadataEnricher(BaseTransformer): - """Enrich items with vendor and metadata information. - - This transformer adds a ``vendor`` field and a ``metadata`` dictionary - containing a ``processed_at`` timestamp to each item. - """ - - def __init__(self, vendor: str) -> None: - """Initialize the MetadataEnricher. - - Args: - vendor: The vendor name to attach to each item. - """ - self.vendor = vendor - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Add vendor and metadata information to an item. - - Args: - item: A dictionary representing a single configuration or URL entry. - - Returns: - The transformed dictionary containing the ``vendor`` field and - a ``metadata.processed_at`` timestamp. - """ - item["vendor"] = self.vendor - if "metadata" not in item: - item["metadata"] = {} - - item["metadata"]["processed_at"] = datetime.utcnow().isoformat() - - return item From 75944c77a057c28879009ddc65d69d4c50d2ef15 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:06:58 +0200 Subject: [PATCH 27/61] Delete transformers/category_mapper.py --- transformers/category_mapper.py | 46 --------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 transformers/category_mapper.py diff --git a/transformers/category_mapper.py b/transformers/category_mapper.py deleted file mode 100644 index 3010f14..0000000 --- a/transformers/category_mapper.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Category mapping transformer. - -This module defines a transformer responsible for mapping category -identifiers between vendor-specific representations and the universal -data model. -""" - -from typing import Any -from typing import Dict - -from .base_transformer import BaseTransformer - - -class CategoryMapper(BaseTransformer): - """Map category identifiers between vendor and universal models. - - This transformer replaces the ``category_id`` field of an item using - a predefined mapping dictionary. If the category is not found in the - mapping, it is left unchanged. - """ - - def __init__(self, category_map: Dict[str, str]) -> None: - """Initialize the CategoryMapper. - - Args: - category_map: A dictionary mapping source category identifiers - to destination category identifiers. - """ - self.category_map = category_map - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Transform an item's category identifier using the category map. - - Args: - item: A dictionary representing a single rule or configuration - entry containing a ``category_id`` field. - - Returns: - The transformed item with its ``category_id`` field mapped - according to the configured category map. - """ - category_id = item.get("category_id") - if category_id in self.category_map: - item["category_id"] = self.category_map[category_id] - - return item From 45bb58ec5d0250ec4258efcb7e61a88bcf0f400d Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:07:11 +0200 Subject: [PATCH 28/61] Delete transformers/base_transformer.py --- transformers/base_transformer.py | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100644 transformers/base_transformer.py diff --git a/transformers/base_transformer.py b/transformers/base_transformer.py deleted file mode 100644 index 04bd40b..0000000 --- a/transformers/base_transformer.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Base transformer definition. - -This module defines the abstract base class used by all transformers -in the transformation pipeline. -""" - -from abc import ABC -from abc import abstractmethod -from typing import Any -from typing import Dict - - -class BaseTransformer(ABC): - """Define the interface for all transformers. - - All concrete transformers must implement the ``transform`` method, - which takes a single dictionary item and returns a transformed - dictionary. - """ - - @abstractmethod - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Transform a single dictionary item. - - Args: - item: A dictionary representing a single configuration or - URL entry. - - Returns: - A transformed dictionary. - """ - raise NotImplementedError From d6f13eb79cc4f9f70d46028375d499bf44199d95 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:07:29 +0200 Subject: [PATCH 29/61] Delete transformers/action_mapper.py --- transformers/action_mapper.py | 45 ----------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 transformers/action_mapper.py diff --git a/transformers/action_mapper.py b/transformers/action_mapper.py deleted file mode 100644 index fc6cb9b..0000000 --- a/transformers/action_mapper.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Action mapping transformer. - -This module defines a transformer responsible for mapping action values -between vendor-specific representations and the universal data model. -""" - -from typing import Any -from typing import Dict - -from .base_transformer import BaseTransformer - - -class ActionMapper(BaseTransformer): - """Map action values between vendor and universal models. - - This transformer replaces the ``action`` field of an item using a - predefined mapping dictionary. If the action is not found in the - mapping, it is left unchanged. - """ - - def __init__(self, action_map: Dict[str, str]) -> None: - """Initialize the ActionMapper. - - Args: - action_map: A dictionary mapping source action values to - destination action values. - """ - self.action_map = action_map - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Transform an item's action field using the action mapping. - - Args: - item: A dictionary representing a single rule or configuration - entry containing an ``action`` field. - - Returns: - The transformed item with its ``action`` field mapped according - to the configured action map. - """ - action = item.get("action") - if action in self.action_map: - item["action"] = self.action_map[action] - - return item From df7890fed855e63748e69fb48be09864dcdeb9e5 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:08:00 +0200 Subject: [PATCH 30/61] Delete transformers/vendors directory --- transformers/vendors/fortinet_transformer.py | 82 ------- transformers/vendors/netskope_transformer.py | 236 ------------------- transformers/vendors/prisma_transformer.py | 54 ----- transformers/vendors/zscaler_transformer.py | 54 ----- 4 files changed, 426 deletions(-) delete mode 100644 transformers/vendors/fortinet_transformer.py delete mode 100644 transformers/vendors/netskope_transformer.py delete mode 100644 transformers/vendors/prisma_transformer.py delete mode 100644 transformers/vendors/zscaler_transformer.py diff --git a/transformers/vendors/fortinet_transformer.py b/transformers/vendors/fortinet_transformer.py deleted file mode 100644 index cd17d34..0000000 --- a/transformers/vendors/fortinet_transformer.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Fortinet transformation pipelines. - -This module defines action, category, and type mappings for Fortinet -URL filtering configurations, as well as transformation pipelines -to convert between Fortinet-specific and universal data models. -""" - -import jmespath - -from transformers.action_mapper import ActionMapper -from transformers.base_transformer import BaseTransformer -from transformers.category_mapper import CategoryMapper -from transformers.metadata_enricher import MetadataEnricher -from transformers.pattern_normalizer import PatternNormalizer -from transformers.type_mapper import TypeMapper - -# ---------------- FORTINET MAPPINGS ---------------- - -FORTINET_ACTION_MAP = { - "block": "block", - "allow": "allow", - "monitor": "monitor", -} - -FORTINET_CATEGORY_MAP = { - "3": "malware", - "4": "phishing", - "5": "gambling", - "default": "uncategorized", -} - -FORTINET_TYPE_MAP = { - "simple": "literal", - "wildcard": "wildcard", - "regex": "regex", - "substring": "substring", -} - - -# ---------------- FORTINET PIPELINES ---------------- - -VENDOR_TO_UNIVERSAL_PIPELINES = [ - ActionMapper(FORTINET_ACTION_MAP), - PatternNormalizer(), - TypeMapper(FORTINET_TYPE_MAP), - CategoryMapper(FORTINET_CATEGORY_MAP), - MetadataEnricher("fortinet"), -] - -UNIVERSAL_TO_VENDOR_PIPELINES = [ - ActionMapper({value: key for key, value in FORTINET_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({value: key for key, value in FORTINET_TYPE_MAP.items()}), - CategoryMapper({value: key for key, value in FORTINET_CATEGORY_MAP.items()}), - MetadataEnricher("fortinet"), -] - -JMESPATH_FLATTEN_URLS = """ -*[?modify_type!='Deleted'].*.data_urls.*.{ - pattern: url, - action: `allow`, - category_id: 'Uncategorized', - list_name: @.name, - list_id: @.object_id, - type: @.data_type -} -""" - -def flatten_fortinet_jmespath(url_lists: dict) -> list[dict]: - """ - Flatten Fortinet JSON using JMESPath. - - Args: - url_lists: Nested Fortinet JSON URL list. - - Returns: - Flat list of URL entries suitable for transformer input. - """ - result = jmespath.search(JMESPATH_FLATTEN_URLS, url_lists) - # jmespath returns a nested list, flatten if needed - flat_result = [item for sublist in result for item in sublist] if result else [] - return flat_result diff --git a/transformers/vendors/netskope_transformer.py b/transformers/vendors/netskope_transformer.py deleted file mode 100644 index ba3088e..0000000 --- a/transformers/vendors/netskope_transformer.py +++ /dev/null @@ -1,236 +0,0 @@ -"""Netskope transformation pipelines and pattern normalization. - -This module defines transformers and mappings required to convert -Netskope URL list configurations to and from the universal data model. -""" -import re -from typing import Any -from typing import Dict -from typing import List -from typing import Optional - -import jmespath - -from transformers.action_mapper import ActionMapper -from transformers.base_transformer import BaseTransformer -from transformers.category_mapper import CategoryMapper -from transformers.metadata_enricher import MetadataEnricher -from transformers.pattern_normalizer import PatternNormalizer -from transformers.type_mapper import TypeMapper - -JMESPATH_NETSKOPE = """ -values(@)[?modify_type!='Deleted'].{ - list_name: name, - list_id: object_id, - type: data_type, - urls: values(data_urls) -} -""" - -def flatten_netskope_jmespath(url_lists: dict) -> list[dict]: - """Flatten the structure using jmespath.""" - extracted = jmespath.search(JMESPATH_NETSKOPE, url_lists) or [] - flat = [] - - for lst in extracted: - for entry in lst.get("urls", []): - url = entry.get("url") - if not url: - continue - flat.append({ - "pattern": url, - "action": "allow", - "category_id": "Uncategorized", - "list_name": lst["list_name"], - "list_id": str(lst["list_id"]), - "type": lst["type"] - }) - return flat - -class NetskopePatternNormalizer(BaseTransformer): - """Normalize Netskope URL patterns for vendor compatibility. - - This transformer converts universal URL patterns into Netskope- - compatible formats: - - - ``literal`` / ``exact`` patterns are preserved. - - ``wildcard`` patterns are converted into regex. - - ``regex`` patterns are passed through unchanged. - """ - - def wildcard_to_regex(self, pattern: str) -> str: - r"""Convert a wildcard domain pattern to a regex. - - Example: - ``*.example.com`` → ``^([^.]+\.)*example\.com$`` - - Args: - pattern: A wildcard URL pattern. - - Returns: - A regex representation of the wildcard pattern. - """ - if pattern.startswith("*."): - domain = pattern[2:].replace(".", r"\.") - return rf"^([^.]+\.)*{domain}$" - - return pattern - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Transform a URL item into a Netskope-compatible pattern. - - Args: - item: A universal URL dictionary. - - Returns: - The transformed dictionary with Netskope pattern semantics. - """ - raw_pattern = item.get("pattern", "") - universal_type = item.get("type", "literal") - - if universal_type in ("literal", "exact"): - item["pattern"] = raw_pattern - item["netskope_type"] = "exact" - - elif universal_type in ("wildcard", "regex"): - item["pattern"] = self.wildcard_to_regex(raw_pattern) - item["netskope_type"] = "regex" - - else: - item["pattern"] = raw_pattern - item["netskope_type"] = "exact" - - return item - - def transform_list( - self, - items: List[Dict[str, Any]], - ) -> List[Dict[str, Any]]: - """Transform a list of URL items. - - Args: - items: A list of universal URL dictionaries. - - Returns: - A list of Netskope-compatible URL dictionaries. - """ - return [self.transform(item) for item in items] - - -class NetskopePatternDenormalizer(BaseTransformer): - """Convert Netskope patterns back to universal model patterns.""" - - def regex_to_wildcard(self, pattern: str) -> Optional[str]: - r"""Convert a Netskope regex pattern back to wildcard format. - - Example: - ``^([^.]+\\.)*example\\.com$`` → ``*.example.com`` - - Args: - pattern: A Netskope regex pattern. - - Returns: - A wildcard pattern if conversion is possible, otherwise ``None``. - """ - wildcard_regex = ( - r"^\^\(\[\^\.\]\+\\\.\)\*(.+)\\\.([a-zA-Z0-9\-]+)\$$" - ) - match = re.match(wildcard_regex, pattern) - - if match: - domain = f"{match.group(1)}.{match.group(2)}" - return f"*.{domain}" - - return None - - def is_regex(self, pattern: str) -> bool: - """Determine whether a pattern contains regex syntax. - - Args: - pattern: A URL pattern string. - - Returns: - ``True`` if the pattern appears to be a regex, otherwise ``False``. - """ - regex_markers = ("^", "$", "(", ")", "[", "]", "+", "?", "|", "{", "}") - return any(marker in pattern for marker in regex_markers) - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Transform a Netskope URL item into a universal-compatible form. - - Args: - item: A Netskope URL dictionary. - - Returns: - A universal URL dictionary. - """ - pattern = item.get("pattern", "").replace("\\\\", "\\") - - if pattern.startswith("*.") and pattern.count("*") == 1: - item["type"] = "wildcard" - - elif "*" in pattern: - item["type"] = "regex" - - elif self.is_regex(pattern): - wildcard = self.regex_to_wildcard(pattern) - if wildcard: - item["type"] = "wildcard" - pattern = wildcard - else: - item["type"] = "regex" - - else: - item["type"] = "exact" - - item["pattern"] = pattern - item.pop("netskope_type", None) - - return item - - -# ---------------- NETSKOPE MAPPINGS ---------------- - -NETSKOPE_ACTION_MAP = { - "block": "deny", - "allow": "allow", - "monitor": "allow", -} - -NETSKOPE_CATEGORY_MAP = { - "malware": "malware", - "phishing": "phishing", - "gambling": "gambling", - "uncategorized": "uncategorized", -} - -NETSKOPE_TO_UNIVERSAL_TYPE_MAP = { - "exact": "literal", - "regex": "regex", -} - -UNIVERSAL_TO_NETSKOPE_TYPE_MAP = { - "literal": "exact", - "regex": "regex", - "wildcard": "regex", - "substring": "regex", -} - - -# ---------------- NETSKOPE PIPELINES ---------------- - -VENDOR_TO_UNIVERSAL_PIPELINES = [ - ActionMapper(NETSKOPE_ACTION_MAP), - TypeMapper(NETSKOPE_TO_UNIVERSAL_TYPE_MAP), - NetskopePatternDenormalizer(), - CategoryMapper(NETSKOPE_CATEGORY_MAP), - MetadataEnricher("netskope"), -] - -UNIVERSAL_TO_VENDOR_PIPELINES = [ - ActionMapper({value: key for key, value in NETSKOPE_ACTION_MAP.items()}), - TypeMapper(UNIVERSAL_TO_NETSKOPE_TYPE_MAP), - NetskopePatternNormalizer(), - CategoryMapper({value: key for key, value in NETSKOPE_CATEGORY_MAP.items()}), - MetadataEnricher("netskope"), -] diff --git a/transformers/vendors/prisma_transformer.py b/transformers/vendors/prisma_transformer.py deleted file mode 100644 index 87ec812..0000000 --- a/transformers/vendors/prisma_transformer.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Prisma transformation pipelines. - -This module defines action, category, and type mappings for Prisma -URL filtering configurations, along with transformation pipelines -to convert between Prisma-specific and universal data models. -""" - -from transformers.action_mapper import ActionMapper -from transformers.base_transformer import BaseTransformer -from transformers.category_mapper import CategoryMapper -from transformers.metadata_enricher import MetadataEnricher -from transformers.pattern_normalizer import PatternNormalizer -from transformers.type_mapper import TypeMapper - -# ---------------- PRISMA MAPPINGS ---------------- - -PRISMA_ACTION_MAP = { - "block": "deny", - "allow": "allow", - "monitor": "alert", -} - -PRISMA_CATEGORY_MAP = { - "malware": "malware", - "phishing": "phishing", - "gambling": "gambling", - "uncategorized": "uncategorized", -} - -PRISMA_TYPE_MAP = { - "simple": "literal", - "wildcard": "wildcard", - "regex": "regex", - "substring": "substring", -} - - -# ---------------- PRISMA PIPELINES ---------------- - -VENDOR_TO_UNIVERSAL_PIPELINES = [ - ActionMapper(PRISMA_ACTION_MAP), - PatternNormalizer(), - TypeMapper(PRISMA_TYPE_MAP), - CategoryMapper(PRISMA_CATEGORY_MAP), - MetadataEnricher("prisma"), -] - -UNIVERSAL_TO_VENDOR_PIPELINES = [ - ActionMapper({value: key for key, value in PRISMA_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({value: key for key, value in PRISMA_TYPE_MAP.items()}), - CategoryMapper({value: key for key, value in PRISMA_CATEGORY_MAP.items()}), - MetadataEnricher("prisma"), -] diff --git a/transformers/vendors/zscaler_transformer.py b/transformers/vendors/zscaler_transformer.py deleted file mode 100644 index 600cb85..0000000 --- a/transformers/vendors/zscaler_transformer.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Zscaler transformation pipelines. - -This module defines action, category, and type mappings for Zscaler -URL filtering configurations, along with transformation pipelines -to convert between Zscaler-specific and universal data models. -""" - -from transformers.action_mapper import ActionMapper -from transformers.base_transformer import BaseTransformer -from transformers.category_mapper import CategoryMapper -from transformers.metadata_enricher import MetadataEnricher -from transformers.pattern_normalizer import PatternNormalizer -from transformers.type_mapper import TypeMapper - -# ---------------- ZSCALER MAPPINGS ---------------- - -ZSCALER_ACTION_MAP = { - "block": "BLOCK", - "allow": "ALLOW", - "monitor": "MONITOR", -} - -ZSCALER_CATEGORY_MAP = { - "malware": "malware", - "phishing": "phishing", - "gambling": "gambling", - "uncategorized": "uncategorized", -} - -ZSCALER_TYPE_MAP = { - "STRING": "literal", - "WILDCARD": "wildcard", - "REGEX": "regex", -} - - -# ---------------- ZSCALER PIPELINES ---------------- - -VENDOR_TO_UNIVERSAL_PIPELINES = [ - ActionMapper(ZSCALER_ACTION_MAP), - PatternNormalizer(), - TypeMapper(ZSCALER_TYPE_MAP), - CategoryMapper(ZSCALER_CATEGORY_MAP), - MetadataEnricher("zscaler"), -] - -UNIVERSAL_TO_VENDOR_PIPELINES = [ - ActionMapper({value: key for key, value in ZSCALER_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({value: key for key, value in ZSCALER_TYPE_MAP.items()}), - CategoryMapper({value: key for key, value in ZSCALER_CATEGORY_MAP.items()}), - MetadataEnricher("zscaler"), -] - From 80779d87d8826be7b157c38cf41c74240857b11a Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:08:37 +0200 Subject: [PATCH 31/61] Delete transformers/shared directory --- transformers/shared/exceptions.py | 24 ------------------------ transformers/shared/logging.py | 24 ------------------------ transformers/shared/utils.py | 24 ------------------------ 3 files changed, 72 deletions(-) delete mode 100644 transformers/shared/exceptions.py delete mode 100644 transformers/shared/logging.py delete mode 100644 transformers/shared/utils.py diff --git a/transformers/shared/exceptions.py b/transformers/shared/exceptions.py deleted file mode 100644 index 3d282c8..0000000 --- a/transformers/shared/exceptions.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -exceptions base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" diff --git a/transformers/shared/logging.py b/transformers/shared/logging.py deleted file mode 100644 index 40d44e8..0000000 --- a/transformers/shared/logging.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -logging base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" diff --git a/transformers/shared/utils.py b/transformers/shared/utils.py deleted file mode 100644 index 31d119f..0000000 --- a/transformers/shared/utils.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -utils base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" From a676f6c6fb3426a9ea7d12ba9b77c4578262ac6d Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:08:47 +0200 Subject: [PATCH 32/61] Delete transformers/core directory --- transformers/core/base.py | 24 ------------------------ transformers/core/enums/enums.py | 24 ------------------------ transformers/core/models/models.py | 24 ------------------------ 3 files changed, 72 deletions(-) delete mode 100644 transformers/core/base.py delete mode 100644 transformers/core/enums/enums.py delete mode 100644 transformers/core/models/models.py diff --git a/transformers/core/base.py b/transformers/core/base.py deleted file mode 100644 index a3031dc..0000000 --- a/transformers/core/base.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Base transformer definitions. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" diff --git a/transformers/core/enums/enums.py b/transformers/core/enums/enums.py deleted file mode 100644 index 79b9fab..0000000 --- a/transformers/core/enums/enums.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -enums base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" diff --git a/transformers/core/models/models.py b/transformers/core/models/models.py deleted file mode 100644 index 365a758..0000000 --- a/transformers/core/models/models.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -models base class. - -Purpose: - - -Context: - Part of the use case within the Unified Policy Transformation Framework. - -Responsibilities: - - - - - - - -Notes: - - Auto-generated placeholder module. - - Extend implementation as needed. - -Author: - - -Created: - -""" From a519b3cadd4df9b7ad1a786e8eb8e6f7171c25f3 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:16:44 +0200 Subject: [PATCH 33/61] Update __init__.py --- transformers/domains/url/vendors/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/transformers/domains/url/vendors/__init__.py b/transformers/domains/url/vendors/__init__.py index 53f13d3..cbcd840 100644 --- a/transformers/domains/url/vendors/__init__.py +++ b/transformers/domains/url/vendors/__init__.py @@ -1,3 +1,9 @@ +""" +Transformers package initialization. + +This package provides core functionality and public APIs for the transformers library. +""" + from . import fortinet # Optional: List of vendors for easy iteration in orchestration logic From c13b4e4be4646781076d4430e429a2c4757b88c9 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:16:57 +0200 Subject: [PATCH 34/61] Update __init__.py --- transformers/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index c80e515..93236b6 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1 +1,5 @@ -# Mark as package +""" +Transformers package initialization. + +This package provides core functionality and public APIs for the transformers library. +""" From ba913df0c7dcd5fc5f84c12f48fb41f3d399e96b Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:17:18 +0200 Subject: [PATCH 35/61] Update __init__.py --- transformers/domains/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/transformers/domains/__init__.py b/transformers/domains/__init__.py index 2996f29..04ff6eb 100644 --- a/transformers/domains/__init__.py +++ b/transformers/domains/__init__.py @@ -1 +1,7 @@ +""" +Transformers package initialization. + +This package provides core functionality and public APIs for the transformers library. +""" + # Sub-package for different logic domains (url, firewall, etc.) From f5ae3e3ebb33f9889ab17d6963b4a446a80f80f2 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:17:34 +0200 Subject: [PATCH 36/61] Update __init__.py --- transformers/domains/url/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/transformers/domains/url/__init__.py b/transformers/domains/url/__init__.py index 69badd6..1cbf6e7 100644 --- a/transformers/domains/url/__init__.py +++ b/transformers/domains/url/__init__.py @@ -1 +1,7 @@ +""" +Transformers package initialization. + +This package provides core functionality and public APIs for the transformers library. +""" + # url domain package From 45b6574c160a3f84dc2726bf13e0b8b7a1db3f0d Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:18:11 +0200 Subject: [PATCH 37/61] Update __init__.py --- transformers/framework/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/transformers/framework/__init__.py b/transformers/framework/__init__.py index 5bb534f..93236b6 100644 --- a/transformers/framework/__init__.py +++ b/transformers/framework/__init__.py @@ -1 +1,5 @@ -# package +""" +Transformers package initialization. + +This package provides core functionality and public APIs for the transformers library. +""" From d3331c7e03a28d9338fb9c8444ee2604ec468b8f Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:18:26 +0200 Subject: [PATCH 38/61] Update __init__.py --- transformers/framework/udm_transformers/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/transformers/framework/udm_transformers/__init__.py b/transformers/framework/udm_transformers/__init__.py index 5bb534f..93236b6 100644 --- a/transformers/framework/udm_transformers/__init__.py +++ b/transformers/framework/udm_transformers/__init__.py @@ -1 +1,5 @@ -# package +""" +Transformers package initialization. + +This package provides core functionality and public APIs for the transformers library. +""" From c298a9072cc4987068f54dd112524cccac8dac33 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:22:38 +0200 Subject: [PATCH 39/61] Refactor model docstrings and field formatting Updated docstrings for clarity and consistency in the models. Reformatted field definitions for better readability. --- transformers/domains/url/models.py | 70 +++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/transformers/domains/url/models.py b/transformers/domains/url/models.py index 0d50bab..13ca238 100644 --- a/transformers/domains/url/models.py +++ b/transformers/domains/url/models.py @@ -7,6 +7,7 @@ across heterogeneous security platforms. [cite: 51-52, 170] Design Principles: + - Domain-Level Logic: Operates purely on domain concepts. [cite: 99] - Vendor-Agnostic: No vendor-specific logic is contained here. [cite: 100] - Strong Typing: Enforces RFC-compliant formatting and normalization. [cite: 173] @@ -19,48 +20,77 @@ class Category(BaseModel): """ - Represents a normalized category entity with a stable identifier - and taxonomic classification. + Represents a normalized category entity. + + This includes a stable identifier and taxonomic classification. """ + id: str = Field(..., description="Internal unique identifier for the category") name: str = Field(..., description="Human-readable name of the category") type: Literal["standard", "custom"] = Field( ..., - description="Distinguishes between system-standard and user-defined categories " + description="Distinguishes between system-standard and user-defined categories", ) class Metadata(BaseModel): """ - Extensible container for enrichment data, such as timestamps and source info. + Extensible container for enrichment data. + + This includes timestamps, source information, and optional metadata fields. """ - processed_at: datetime = Field(..., description="Timestamp of when the record was processed") - source: Optional[str] = Field(None, description="The origin system of the data") - additional_info: Optional[dict] = Field(None, description="Placeholder for custom metadata expansion") + + processed_at: datetime = Field( + ..., description="Timestamp of when the record was processed" + ) + source: Optional[str] = Field( + None, description="The origin system of the data" + ) + additional_info: Optional[dict] = Field( + None, description="Placeholder for custom metadata expansion" + ) class URL_UDM(BaseModel): """ - The Unified Data Model for URL entities. + Unified Data Model for URL entities. This model serves as the source of truth for processing, independent - of any external vendor system. + of any external vendor system. """ - # Performance optimization for Pydantic v2 + model_config = ConfigDict(populate_by_name=True) - pattern: str = Field(..., description="The URL pattern (literal, wildcard, or regex) ") - type: Literal["literal", "wildcard", "regex"] = Field(..., description="The syntax type of the pattern") - action: Literal["allow", "block", "monitor"] = Field(..., description="Standardized enforcement action ") - status: Literal["enable", "disable"] = Field(..., description="Operational status of the rule") - url_list_id: str = Field(..., description="Unique ID for the parent URL list ") - url_list_name: str = Field(..., description="Human-readable name of the URL list") + pattern: str = Field( + ..., description="The URL pattern (literal, wildcard, or regex)" + ) + type: Literal["literal", "wildcard", "regex"] = Field( + ..., description="The syntax type of the pattern" + ) + action: Literal["allow", "block", "monitor"] = Field( + ..., description="Standardized enforcement action" + ) + status: Literal["enable", "disable"] = Field( + ..., description="Operational status of the rule" + ) + url_list_id: str = Field( + ..., description="Unique ID for the parent URL list" + ) + url_list_name: str = Field( + ..., description="Human-readable name of the URL list" + ) categories: List[Category] = Field( default_factory=list, - description="Merged array of standard and custom categories " + description="Merged array of standard and custom categories", ) - vendor: Optional[str] = Field(None, description="Original vendor for traceability purposes ]") - metadata: Optional[Metadata] = Field(None, description="Processing metadata and timestamps") - notes: Optional[str] = Field(None, description="Optional justifications or comments ") + vendor: Optional[str] = Field( + None, description="Original vendor for traceability purposes" + ) + metadata: Optional[Metadata] = Field( + None, description="Processing metadata and timestamps" + ) + notes: Optional[str] = Field( + None, description="Optional justifications or comments" + ) From 8b2200accc7712c8c50448146a09d94ad803a2e1 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:26:24 +0200 Subject: [PATCH 40/61] Fix docstring formatting in models.py Removed trailing period from module docstring and adjusted formatting. --- transformers/domains/url/models.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/transformers/domains/url/models.py b/transformers/domains/url/models.py index 13ca238..f4239ef 100644 --- a/transformers/domains/url/models.py +++ b/transformers/domains/url/models.py @@ -1,16 +1,14 @@ """ -URL Domain Models - Unified Data Model (UDM) +URL Domain Models - Unified Data Model (UDM). This module defines the canonical schema for URLs, URL collections, and -categories within the URL domain. It abstracts vendor-specific constructs -into a standardized representation to enable consistent policy handling -across heterogeneous security platforms. [cite: 51-52, 170] +categories within the URL domain. Design Principles: -- Domain-Level Logic: Operates purely on domain concepts. [cite: 99] -- Vendor-Agnostic: No vendor-specific logic is contained here. [cite: 100] -- Strong Typing: Enforces RFC-compliant formatting and normalization. [cite: 173] +- Domain-Level Logic: Operates purely on domain concepts. +- Vendor-Agnostic: No vendor-specific logic is contained here. +- Strong Typing: Enforces RFC-compliant formatting and normalization. """ from typing import List, Literal, Optional From 3f5dada4343754333835902ad8893eda2d127408 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:29:36 +0200 Subject: [PATCH 41/61] Update models.py --- transformers/domains/url/models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/transformers/domains/url/models.py b/transformers/domains/url/models.py index f4239ef..5ec6dba 100644 --- a/transformers/domains/url/models.py +++ b/transformers/domains/url/models.py @@ -11,9 +11,14 @@ - Strong Typing: Enforces RFC-compliant formatting and normalization. """ -from typing import List, Literal, Optional from datetime import datetime -from pydantic import BaseModel, Field, ConfigDict +from typing import List +from typing import Literal +from typing import Optional + +from pydantic import BaseModel +from pydantic import ConfigDict +from pydantic import Field class Category(BaseModel): From 2cd34f38e1a30216afe93165b9ca32c4736f3d44 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:33:02 +0200 Subject: [PATCH 42/61] Clean up vendor imports in __init__.py Removed import of fortinet and updated __all__ list. --- transformers/domains/url/vendors/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/transformers/domains/url/vendors/__init__.py b/transformers/domains/url/vendors/__init__.py index cbcd840..93236b6 100644 --- a/transformers/domains/url/vendors/__init__.py +++ b/transformers/domains/url/vendors/__init__.py @@ -3,8 +3,3 @@ This package provides core functionality and public APIs for the transformers library. """ - -from . import fortinet - -# Optional: List of vendors for easy iteration in orchestration logic -__all__ = ["fortinet"] From d7a836485eec9bd3c9046d71494b4408e2b2d7ff Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:37:38 +0200 Subject: [PATCH 43/61] Refactor Fortinet module for improved clarity --- transformers/domains/url/vendors/fortinet.py | 165 +++++++++---------- 1 file changed, 81 insertions(+), 84 deletions(-) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index e3f6fc1..1f0a161 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -1,34 +1,33 @@ """ -Fortinet URL Domain Integration +Fortinet URL Domain Integration. -This module implements the Transformer, Mapper, and Exporter for Fortinet, -converting between Fortinet-specific configurations and the Pydantic -Unified Data Model (UDM). +This module implements the transformer, mapper, and exporter for Fortinet, +converting between Fortinet-specific configurations and the Unified Data +Model (UDM). """ -import jmespath from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any +from typing import Dict +from typing import List +from typing import Optional + +import jmespath -# Framework imports - Absolute paths +from transformers.domains.url.models import Category +from transformers.domains.url.models import Metadata +from transformers.domains.url.models import URL_UDM from transformers.framework.udm_transformers.action_mapper import ActionMapper from transformers.framework.udm_transformers.category_mapper import CategoryMapper from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer from transformers.framework.udm_transformers.type_mapper import TypeMapper -# Domain Model imports -from transformers.domains.url.models import URL_UDM -from transformers.domains.url.models import Category -from transformers.domains.url.models import Metadata - -# ---------------- FORTINET MAPPINGS ---------------- - FORTINET_ACTION_MAP = { "allow": "allow", "block": "block", "monitor": "monitor", - "exempt": "allow" + "exempt": "allow", } FORTINET_CATEGORY_MAP = { @@ -44,9 +43,6 @@ "regex": "regex", } -# ---------------- EXTRACTION LAYER ---------------- - - JMESPATH_FLATTEN_URLS = """ *.urls.*.{ pattern: url, @@ -57,7 +53,9 @@ } """ -def flatten_fortinet_jmespath(raw_data): + +def flatten_fortinet_jmespath(raw_data: Dict[str, Any]) -> List[Dict[str, Any]]: + """Flatten nested Fortinet URL data into normalized records.""" flat = [] for _, url_list in raw_data.items(): @@ -65,40 +63,41 @@ def flatten_fortinet_jmespath(raw_data): list_name = url_list["filter_name"] for _, item in url_list["urls"].items(): - flat.append({ - "pattern": item["url"], - "action": item["action"], - "status": item["status"], - "type": item["type"], - "url_id": item["url_id"], - "list_id": list_id, - "list_name": list_name, - "category_id": "Uncategorized" - }) + flat.append( + { + "pattern": item["url"], + "action": item["action"], + "status": item["status"], + "type": item["type"], + "url_id": item["url_id"], + "list_id": list_id, + "list_name": list_name, + "category_id": "Uncategorized", + } + ) return flat -# ---------------- MAPPER & EXPORTER ---------------- class FortinetMapper: - """ - Handles semantic alignment and Pydantic UDM instantiation. - """ + """Map transformed dictionaries into URL_UDM instances.""" def to_udm(self, item: Dict[str, Any]) -> URL_UDM: - """ - Converts a transformed dictionary into a validated URL_UDM instance. - """ - # Map categories to the Category model + """Convert a transformed dictionary into a validated URL_UDM.""" cat_id = item.get("category_id", "uncategorized") categories = [ - Category(id=cat_id, name=cat_id.capitalize(), type="standard") + Category( + id=cat_id, + name=cat_id.capitalize(), + type="standard", + ) ] - # Construct the Metadata model - # MetadataEnricher provides the ISO timestamp string meta = Metadata( - processed_at=datetime.fromisoformat(item["metadata"]["processed_at"]), + processed_at=datetime.fromisoformat( + item["metadata"]["processed_at"] + ), + source=item["metadata"].get("source"), ) return URL_UDM( @@ -111,52 +110,58 @@ def to_udm(self, item: Dict[str, Any]) -> URL_UDM: categories=categories, vendor=item["vendor"], metadata=meta, - notes=item.get("notes") + notes=item.get("notes"), ) + class FortinetExporter: - """ - Universal Model -> Fortinet Format. - """ + """Export URL_UDM records into Fortinet format.""" + def transform(self, udm: URL_UDM) -> Dict[str, Any]: - """ - Reconstructs Fortinet-specific pattern and type syntax. - """ - # Reverse mapping for Type [cite: 347] - reverse_type_map = {v: k for k, v in FORTINET_TYPE_MAP.items()} + """Reconstruct Fortinet-specific pattern and type syntax.""" + reverse_type_map = { + value: key for key, value in FORTINET_TYPE_MAP.items() + } return { "url": udm.pattern, - "type": reverse_type_map.get(udm.type, "simple") + "type": reverse_type_map.get(udm.type, "simple"), } -def run_universal_to_fortinet_pipeline(records: List[URL_UDM]) -> List[dict]: + +def run_universal_to_fortinet_pipeline( + records: List[URL_UDM], +) -> List[Dict[str, Any]]: + """Transform universal records into Fortinet export records.""" output = [] - for r in records: - output.append({ - "pattern": r.pattern, - "type": r.type, - "action": r.action, - "list_id": r.url_list_id, - "list_name": r.url_list_name - }) + for record in records: + output.append( + { + "pattern": record.pattern, + "type": record.type, + "action": record.action, + "list_id": record.url_list_id, + "list_name": record.url_list_name, + } + ) return output -def export_fortinet_json(records: List[dict]) -> dict: +def export_fortinet_json(records: List[Dict[str, Any]]) -> Dict[str, Any]: + """Convert flat Fortinet records into grouped Fortinet JSON.""" grouped = {} counters = {} - for r in records: - key = r["list_id"] + for record in records: + key = record["list_id"] if key not in grouped: grouped[key] = { - "object_id": r["list_id"], - "filter_name": r["list_name"], - "urls": {} + "object_id": record["list_id"], + "filter_name": record["list_name"], + "urls": {}, } counters[key] = 0 @@ -165,49 +170,41 @@ def export_fortinet_json(records: List[dict]) -> dict: grouped[key]["urls"][idx] = { "url_id": str(counters[key]), - "url": r["pattern"], - "type": r["type"], - "action": r["action"], - "status": "enable" + "url": record["pattern"], + "type": record["type"], + "action": record["action"], + "status": "enable", } return grouped -# ---------------- EXECUTION PIPELINE ---------------- -def run_fortinet_to_universal_pipeline(raw_data: Dict[str, Any]) -> List[URL_UDM]: - """ - Orchestrates the deterministic flow from raw Fortinet data to UDM objects. - """ - # 1. Extraction +def run_fortinet_to_universal_pipeline( + raw_data: Dict[str, Any], +) -> List[URL_UDM]: + """Run the full Fortinet-to-universal transformation pipeline.""" flat_data = flatten_fortinet_jmespath(raw_data) - # 2. Transformation Pipeline steps = [ ActionMapper(FORTINET_ACTION_MAP), PatternNormalizer(), TypeMapper(FORTINET_TYPE_MAP), CategoryMapper(FORTINET_CATEGORY_MAP), - MetadataEnricher("fortinet") + MetadataEnricher("fortinet"), ] mapper = FortinetMapper() udm_records = [] for record in flat_data: - # Apply each modular transformation unit for step in steps: record = step.transform(record) - # 3. Validation & Pydantic Conversion udm_records.append(mapper.to_udm(record)) return udm_records -# ---------------- REGISTRATION ---------------- -# This is what debugPythonScript.py is looking for VENDOR_TO_UNIVERSAL_PIPELINES = { - "fortinet": run_fortinet_to_universal_pipeline + "fortinet": run_fortinet_to_universal_pipeline, } - From 5c68a0a497dcfae09a1f16ced78ea5b73f214434 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:41:57 +0200 Subject: [PATCH 44/61] Update fortinet.py From cba2a64fb47cbe986bbd4dfc8a7d92f1f7e3935f Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:45:12 +0200 Subject: [PATCH 45/61] Update fortinet.py --- transformers/domains/url/vendors/fortinet.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index 1f0a161..fed1c6a 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -6,17 +6,18 @@ Model (UDM). """ +import jmespath + from datetime import datetime from typing import Any from typing import Dict from typing import List from typing import Optional -import jmespath - from transformers.domains.url.models import Category from transformers.domains.url.models import Metadata from transformers.domains.url.models import URL_UDM + from transformers.framework.udm_transformers.action_mapper import ActionMapper from transformers.framework.udm_transformers.category_mapper import CategoryMapper from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher From 6fd5e138fcf97d63fcefc4d9f6b6cef18ae9d353 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:48:38 +0200 Subject: [PATCH 46/61] Update fortinet.py --- transformers/domains/url/vendors/fortinet.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index fed1c6a..bd666b0 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -6,24 +6,22 @@ Model (UDM). """ -import jmespath - from datetime import datetime from typing import Any -from typing import Dict from typing import List from typing import Optional +import jmespath + +from transformers.domains.url.models import URL_UDM from transformers.domains.url.models import Category from transformers.domains.url.models import Metadata -from transformers.domains.url.models import URL_UDM from transformers.framework.udm_transformers.action_mapper import ActionMapper from transformers.framework.udm_transformers.category_mapper import CategoryMapper from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer from transformers.framework.udm_transformers.type_mapper import TypeMapper - FORTINET_ACTION_MAP = { "allow": "allow", "block": "block", From 28a89f978339909887b37fdb3ab759f3058a3168 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:49:52 +0200 Subject: [PATCH 47/61] Update fortinet.py --- transformers/domains/url/vendors/fortinet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index bd666b0..832b75d 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -22,6 +22,7 @@ from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer from transformers.framework.udm_transformers.type_mapper import TypeMapper + FORTINET_ACTION_MAP = { "allow": "allow", "block": "block", From 7f35c60502dc310d415fee72ffe50581dbd2ed5f Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 15:56:53 +0200 Subject: [PATCH 48/61] Refactor imports in fortinet.py Refactor import statements for better readability. --- transformers/domains/url/vendors/fortinet.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index 832b75d..bc443aa 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -16,11 +16,12 @@ from transformers.domains.url.models import URL_UDM from transformers.domains.url.models import Category from transformers.domains.url.models import Metadata - -from transformers.framework.udm_transformers.action_mapper import ActionMapper -from transformers.framework.udm_transformers.category_mapper import CategoryMapper -from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher -from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer +from transformers.framework.udm_transformers.category_mapper import \ + CategoryMapper +from transformers.framework.udm_transformers.metadata_enricher import \ + MetadataEnricher +from transformers.framework.udm_transformers.pattern_normalizer import \ + PatternNormalizer from transformers.framework.udm_transformers.type_mapper import TypeMapper FORTINET_ACTION_MAP = { From 30bec704db97144dd782b511b0ee0a98f3d31549 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:01:38 +0200 Subject: [PATCH 49/61] Refactor Netskope integration for improved clarity --- transformers/domains/url/vendors/netskope.py | 111 +++++++++++-------- 1 file changed, 63 insertions(+), 48 deletions(-) diff --git a/transformers/domains/url/vendors/netskope.py b/transformers/domains/url/vendors/netskope.py index d106cfc..a720e91 100644 --- a/transformers/domains/url/vendors/netskope.py +++ b/transformers/domains/url/vendors/netskope.py @@ -1,30 +1,29 @@ """ -Netskope URL Domain Integration +Netskope URL Domain Integration. -This module implements the Transformer, Mapper, and Exporter for Netskope, +This module implements the transformer, mapper, and exporter for Netskope, converting between Netskope-specific configurations and the Pydantic Unified Data Model (UDM). """ -import re -import jmespath from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any +from typing import List +from typing import Optional -# Framework imports - Absolute paths -from transformers.framework.udm_transformers.action_mapper import ActionMapper -from transformers.framework.udm_transformers.category_mapper import CategoryMapper -from transformers.framework.udm_transformers.metadata_enricher import MetadataEnricher -from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer -from transformers.framework.udm_transformers.type_mapper import TypeMapper -from transformers.framework.udm_transformers.base_transformer import BaseTransformer +import jmespath -# Domain Model imports from transformers.domains.url.models import URL_UDM from transformers.domains.url.models import Category from transformers.domains.url.models import Metadata +from transformers.framework.udm_transformers.category_mapper import \ + CategoryMapper +from transformers.framework.udm_transformers.metadata_enricher import \ + MetadataEnricher +from transformers.framework.udm_transformers.pattern_normalizer import \ + PatternNormalizer +from transformers.framework.udm_transformers.type_mapper import TypeMapper -# ---------------- NETSKOPE MAPPINGS ---------------- NETSKOPE_ACTION_MAP = { "block": "deny", @@ -44,8 +43,6 @@ "regex": "regex", } -# ---------------- EXTRACTION LAYER ---------------- - JMESPATH_NETSKOPE = """ values(@)[?modify_type!='Deleted'].{ list_name: name, @@ -55,8 +52,9 @@ } """ + def flatten_netskope_jmespath(url_lists: Dict[str, Any]) -> List[Dict[str, Any]]: - """Flatten the structure using jmespath. """ + """Flatten Netskope hierarchical data using JMESPath extraction.""" extracted = jmespath.search(JMESPATH_NETSKOPE, url_lists) or [] flat = [] @@ -65,29 +63,33 @@ def flatten_netskope_jmespath(url_lists: Dict[str, Any]) -> List[Dict[str, Any]] url = entry.get("url") if not url: continue - flat.append({ - "pattern": url, - "action": "allow", - "category_id": "Uncategorized", - "list_name": lst["list_name"], - "list_id": str(lst["list_id"]), - "type": lst["type"] - }) + + flat.append( + { + "pattern": url, + "action": "allow", + "category_id": "Uncategorized", + "list_name": lst["list_name"], + "list_id": str(lst["list_id"]), + "type": lst["type"], + } + ) + return flat -# ---------------- TRANSFORMERS ---------------- class NetskopePatternNormalizer(BaseTransformer): - """Normalize Netskope URL patterns for vendor compatibility.""" + """Normalize Netskope URL patterns for universal compatibility.""" def wildcard_to_regex(self, pattern: str) -> str: - """Convert a wildcard domain pattern to a regex.""" + """Convert wildcard pattern to regex format.""" if pattern.startswith("*."): domain = pattern[2:].replace(".", r"\.") return rf"^([^.]+\.)*{domain}$" return pattern def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Normalize pattern into Netskope-compatible format.""" raw_pattern = item.get("pattern", "") universal_type = item.get("type", "literal") @@ -100,12 +102,15 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: else: item["pattern"] = raw_pattern item["netskope_type"] = "exact" + return item + class NetskopePatternDenormalizer(BaseTransformer): - """Convert Netskope patterns back to universal model patterns.""" + """Convert Netskope patterns back to universal format.""" def regex_to_wildcard(self, pattern: str) -> Optional[str]: + """Attempt to convert regex to wildcard pattern.""" wildcard_regex = r"^\^\(\[\^\.\]\+\\\.\)\*(.+)\\\.([a-zA-Z0-9\-]+)\$$" match = re.match(wildcard_regex, pattern) if match: @@ -114,11 +119,14 @@ def regex_to_wildcard(self, pattern: str) -> Optional[str]: return None def is_regex(self, pattern: str) -> bool: + """Check if a pattern contains regex syntax.""" regex_markers = ("^", "$", "(", ")", "[", "]", "+", "?", "|", "{", "}") return any(marker in pattern for marker in regex_markers) def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Denormalize Netskope pattern into universal format.""" pattern = item.get("pattern", "").replace("\\\\", "\\") + if pattern.startswith("*.") and pattern.count("*") == 1: item["type"] = "wildcard" elif "*" in pattern: @@ -132,23 +140,27 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: item["type"] = "regex" else: item["type"] = "exact" + item["pattern"] = pattern item.pop("netskope_type", None) return item -# ---------------- MAPPER & EXPORTER ---------------- class NetskopeMapper: - """Handles semantic alignment and Pydantic UDM instantiation.""" + """Handles semantic mapping into the Unified Data Model.""" def to_udm(self, item: Dict[str, Any]) -> URL_UDM: - """Converts transformed dictionary into validated URL_UDM instance. """ + """Convert transformed dictionary into URL_UDM instance.""" cat_id = item.get("category_id", "uncategorized") - categories = [Category(id=cat_id, name=cat_id.capitalize(), type="standard")] + categories = [ + Category(id=cat_id, name=cat_id.capitalize(), type="standard") + ] meta = Metadata( - processed_at=datetime.fromisoformat(item["metadata"]["processed_at"]), - source="netskope" + processed_at=datetime.fromisoformat( + item["metadata"]["processed_at"] + ), + source="netskope", ) return URL_UDM( @@ -161,30 +173,34 @@ def to_udm(self, item: Dict[str, Any]) -> URL_UDM: categories=categories, vendor=item["vendor"], metadata=meta, - notes=item.get("notes") + notes=item.get("notes"), ) + class NetskopeExporter: - """Universal Model -> Netskope Format. """ + """Convert UDM objects into Netskope format.""" def transform(self, udm: URL_UDM) -> Dict[str, Any]: - # Implementation would use NetskopePatternNormalizer logic here - pass - -# ---------------- EXECUTION PIPELINE ---------------- - -def run_netskope_to_universal_pipeline(raw_data: Dict[str, Any]) -> List[URL_UDM]: - """Orchestrates the flow from raw Netskope data to UDM objects. """ - # 1. Extraction + """Convert UDM object into Netskope-compatible structure.""" + return { + "pattern": udm.pattern, + "type": udm.type, + "action": udm.action, + } + + +def run_netskope_to_universal_pipeline( + raw_data: Dict[str, Any], +) -> List[URL_UDM]: + """Run full Netskope ingestion pipeline into UDM objects.""" flat_data = flatten_netskope_jmespath(raw_data) - # 2. Transformation Pipeline steps = [ ActionMapper(NETSKOPE_ACTION_MAP), TypeMapper(NETSKOPE_TO_UNIVERSAL_TYPE_MAP), NetskopePatternDenormalizer(), CategoryMapper(NETSKOPE_CATEGORY_MAP), - MetadataEnricher("netskope") + MetadataEnricher("netskope"), ] mapper = NetskopeMapper() @@ -196,4 +212,3 @@ def run_netskope_to_universal_pipeline(raw_data: Dict[str, Any]) -> List[URL_UDM udm_records.append(mapper.to_udm(record)) return udm_records - From 27706601ea6be32c9949d559c0b5068f90ba1ef9 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:06:03 +0200 Subject: [PATCH 50/61] Update fortinet.py --- transformers/domains/url/vendors/fortinet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index bc443aa..f33b977 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -20,10 +20,10 @@ CategoryMapper from transformers.framework.udm_transformers.metadata_enricher import \ MetadataEnricher -from transformers.framework.udm_transformers.pattern_normalizer import \ - PatternNormalizer +from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer from transformers.framework.udm_transformers.type_mapper import TypeMapper + FORTINET_ACTION_MAP = { "allow": "allow", "block": "block", From ee8e65e82d7c56d2c848c49373523940fafaf2e2 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:09:05 +0200 Subject: [PATCH 51/61] Update fortinet.py --- transformers/domains/url/vendors/fortinet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/domains/url/vendors/fortinet.py b/transformers/domains/url/vendors/fortinet.py index f33b977..bc443aa 100644 --- a/transformers/domains/url/vendors/fortinet.py +++ b/transformers/domains/url/vendors/fortinet.py @@ -20,10 +20,10 @@ CategoryMapper from transformers.framework.udm_transformers.metadata_enricher import \ MetadataEnricher -from transformers.framework.udm_transformers.pattern_normalizer import PatternNormalizer +from transformers.framework.udm_transformers.pattern_normalizer import \ + PatternNormalizer from transformers.framework.udm_transformers.type_mapper import TypeMapper - FORTINET_ACTION_MAP = { "allow": "allow", "block": "block", From 7d1f76ada81e7865fb8405925fbb6277e0370852 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:13:29 +0200 Subject: [PATCH 52/61] Update fortinet.py From 37ab1856988058ba37b70cd71dfaf1beaff5746f Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:16:00 +0200 Subject: [PATCH 53/61] Clean up formatting in netskope.py Removed unnecessary newline in netskope.py. --- transformers/domains/url/vendors/netskope.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transformers/domains/url/vendors/netskope.py b/transformers/domains/url/vendors/netskope.py index a720e91..7a7a67e 100644 --- a/transformers/domains/url/vendors/netskope.py +++ b/transformers/domains/url/vendors/netskope.py @@ -24,7 +24,6 @@ PatternNormalizer from transformers.framework.udm_transformers.type_mapper import TypeMapper - NETSKOPE_ACTION_MAP = { "block": "deny", "allow": "allow", From d49f298b188c7f905acbdbb3731ac892bac43d33 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:19:35 +0200 Subject: [PATCH 54/61] Update pipelines.py --- transformers/framework/pipelines.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/framework/pipelines.py b/transformers/framework/pipelines.py index 6967ea8..ac52d08 100644 --- a/transformers/framework/pipelines.py +++ b/transformers/framework/pipelines.py @@ -8,8 +8,8 @@ from typing import Dict from typing import List -from transformers.framework.udm_transformers.base_transformer import BaseTransformer - +from transformers.framework.udm_transformers.base_transformer import \ + BaseTransformer def apply_transformers( items: List[Dict[str, Any]], From 2335457c7cb29cbf27b1f0f572e2f74164a91af0 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:22:42 +0200 Subject: [PATCH 55/61] Update pipelines.py --- transformers/framework/pipelines.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/framework/pipelines.py b/transformers/framework/pipelines.py index ac52d08..46927ee 100644 --- a/transformers/framework/pipelines.py +++ b/transformers/framework/pipelines.py @@ -11,6 +11,7 @@ from transformers.framework.udm_transformers.base_transformer import \ BaseTransformer + def apply_transformers( items: List[Dict[str, Any]], transformers: List[BaseTransformer] From f51fb182b1a1c9f642da6f651ad405f3732360da Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:25:29 +0200 Subject: [PATCH 56/61] Update import statement for BaseTransformer --- transformers/framework/udm_transformers/action_mapper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transformers/framework/udm_transformers/action_mapper.py b/transformers/framework/udm_transformers/action_mapper.py index 111b5a6..279629a 100644 --- a/transformers/framework/udm_transformers/action_mapper.py +++ b/transformers/framework/udm_transformers/action_mapper.py @@ -9,7 +9,9 @@ # Change from: from .base_transformer import BaseTransformer # To the absolute framework path: -from transformers.framework.udm_transformers.base_transformer import BaseTransformer +from transformers.framework.udm_transformers.base_transformer import \ + BaseTransformer + class ActionMapper(BaseTransformer): """Map action values between vendor and universal models. From 91d622f74e9d22dfd2b20b7f09d1f3f105966539 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:25:57 +0200 Subject: [PATCH 57/61] Update category_mapper.py --- transformers/framework/udm_transformers/category_mapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/framework/udm_transformers/category_mapper.py b/transformers/framework/udm_transformers/category_mapper.py index c731a9c..a1d99c0 100644 --- a/transformers/framework/udm_transformers/category_mapper.py +++ b/transformers/framework/udm_transformers/category_mapper.py @@ -9,7 +9,8 @@ from typing import Dict # Restructured to use the absolute path within the Framework layer -from transformers.framework.udm_transformers.base_transformer import BaseTransformer +from transformers.framework.udm_transformers.base_transformer import \ + BaseTransformer class CategoryMapper(BaseTransformer): From c725caf03df185f85181dc2db83ead13bd792562 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:26:18 +0200 Subject: [PATCH 58/61] Update metadata_enricher.py --- transformers/framework/udm_transformers/metadata_enricher.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/framework/udm_transformers/metadata_enricher.py b/transformers/framework/udm_transformers/metadata_enricher.py index 6206e08..39e4d10 100644 --- a/transformers/framework/udm_transformers/metadata_enricher.py +++ b/transformers/framework/udm_transformers/metadata_enricher.py @@ -9,7 +9,8 @@ from typing import Dict # Restructured to use the absolute path within the Framework layer -from transformers.framework.udm_transformers.base_transformer import BaseTransformer +from transformers.framework.udm_transformers.base_transformer import \ + BaseTransformer class MetadataEnricher(BaseTransformer): From 9bc9d8ff406de31b8b90e25869da3471e51c8589 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:26:32 +0200 Subject: [PATCH 59/61] Update pattern_normalizer.py --- transformers/framework/udm_transformers/pattern_normalizer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/framework/udm_transformers/pattern_normalizer.py b/transformers/framework/udm_transformers/pattern_normalizer.py index ed315a5..7a85b25 100644 --- a/transformers/framework/udm_transformers/pattern_normalizer.py +++ b/transformers/framework/udm_transformers/pattern_normalizer.py @@ -9,7 +9,8 @@ from typing import Dict # Restructured to use the absolute path within the Framework layer -from transformers.framework.udm_transformers.base_transformer import BaseTransformer +from transformers.framework.udm_transformers.base_transformer import \ + BaseTransformer class PatternNormalizer(BaseTransformer): From 78db3df9115418f30d02e13e81bc383203dfb792 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:26:52 +0200 Subject: [PATCH 60/61] Update type_mapper.py --- transformers/framework/udm_transformers/type_mapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/framework/udm_transformers/type_mapper.py b/transformers/framework/udm_transformers/type_mapper.py index 8285604..d058d59 100644 --- a/transformers/framework/udm_transformers/type_mapper.py +++ b/transformers/framework/udm_transformers/type_mapper.py @@ -9,7 +9,8 @@ from typing import Dict # Restructured to use the absolute path within the Framework layer -from transformers.framework.udm_transformers.base_transformer import BaseTransformer +from transformers.framework.udm_transformers.base_transformer import \ + BaseTransformer class TypeMapper(BaseTransformer): From 086ea39647d5a99d9877f4037252a4f6fd7f76d6 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Tue, 21 Apr 2026 16:29:40 +0200 Subject: [PATCH 61/61] Update base_transformer.py --- transformers/framework/udm_transformers/base_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/framework/udm_transformers/base_transformer.py b/transformers/framework/udm_transformers/base_transformer.py index e689d2c..161d545 100644 --- a/transformers/framework/udm_transformers/base_transformer.py +++ b/transformers/framework/udm_transformers/base_transformer.py @@ -1,5 +1,5 @@ """ -Base Transformer Definition - Framework Layer +Base Transformer Definition - Framework Layer. This module defines the abstract base class (ABC) used by all transformers within the generic transformation engine. It ensures a consistent interface