From 68f5b01b6744569e09eee272cbe848954803122a Mon Sep 17 00:00:00 2001 From: latifie Date: Tue, 2 Dec 2025 14:06:31 +0100 Subject: [PATCH 1/9] Create transformers.py for URL list migration --- transformers/transformers.py | 258 +++++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 transformers/transformers.py diff --git a/transformers/transformers.py b/transformers/transformers.py new file mode 100644 index 00000000..6081d480 --- /dev/null +++ b/transformers/transformers.py @@ -0,0 +1,258 @@ +""" +transformers.py +A single-file SDK for bidirectional URL-filter migration transformers. + +Supports: +- Vendors: Fortinet, Netskope, Zscaler, Prisma Access +- Universal intermediate model +- Modular basic transformers (action, pattern, category, metadata) +- CLI and unit test support +- Built-in sample data for direct testing +""" + +from typing import List, Dict, Any +from dataclasses import dataclass, field + +# ---------------- UNIVERSAL DATA MODEL ---------------- + +@dataclass +class UniversalURLFilter: + pattern: str + action: str + category: str + type: str = "literal" # literal, wildcard, regex, substring + vendor: str = "" + metadata: Dict[str, Any] = field(default_factory=dict) + +# ---------------- BASIC TRANSFORMERS ---------------- + +class BaseTransformer: + """Abstract transformer""" + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + raise NotImplementedError + +class ActionMapper(BaseTransformer): + def __init__(self, action_map: Dict[str, str]): + self.action_map = action_map + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + item['action'] = self.action_map.get(item.get('action'), item.get('action')) + return item + +class PatternNormalizer(BaseTransformer): + """Keep pattern as-is; do not change type""" + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + item['pattern'] = item.get('pattern', '') + return item + +class TypeMapper(BaseTransformer): + """Map vendor type -> universal type""" + def __init__(self, type_map: Dict[str, str]): + self.type_map = type_map + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + # Always read the original 'type' from vendor config + vendor_type = item.get('type', 'simple') + # Normalize case for Zscaler + vendor_type_normalized = vendor_type.upper() if vendor_type in ['STRING', 'WILDCARD', 'REGEX'] else vendor_type.lower() + # Map to universal type + item['type'] = self.type_map.get(vendor_type_normalized, 'literal') + return item + +class CategoryMapper(BaseTransformer): + """Map vendor category to universal category""" + def __init__(self, category_map: Dict[str, str]): + self.category_map = category_map + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + vendor_cat = str(item.get('category_id', 'default')) # read from input + item['category'] = self.category_map.get(vendor_cat, 'uncategorized') + return item + +class MetadataEnricher(BaseTransformer): + """Add vendor metadata""" + def __init__(self, vendor: str): + self.vendor = vendor + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + item['vendor'] = self.vendor + return item + +# ---------------- PIPELINE EXECUTOR ---------------- + +def apply_transformers(items: List[Dict[str, Any]], transformers: List[BaseTransformer]) -> List[Dict[str, Any]]: + result = [] + for item in items: + for t in transformers: + item = t.transform(item) + result.append(item) + return result + +# ---------------- VENDOR MAPPINGS ---------------- + +FORTINET_ACTION_MAP = {'block': 'block', 'allow': 'allow', 'monitor': 'monitor'} +FORTINET_CATEGORY_MAP = {'3': 'malware', '4': 'phishing', '5': 'gambling', 'default': 'uncategorized'} +FORTINET_TYPE_MAP = { "simple": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} + +NETSKOPE_ACTION_MAP = {'block': 'deny', 'allow': 'allow', 'monitor': 'monitor'} +NETSKOPE_CATEGORY_MAP = {'malware': 'malware', 'phishing': 'phishing', 'gambling': 'gambling', 'uncategorized': 'uncategorized'} +NETSKOPE_TYPE_MAP = { "exact": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} + +ZSCALER_ACTION_MAP = {'block': 'BLOCK', 'allow': 'ALLOW', 'monitor': 'MONITOR'} +ZSCALER_CATEGORY_MAP = {'malware': 'malware', 'phishing': 'phishing', 'gambling': 'gambling', 'uncategorized': 'uncategorized'} +ZSCALER_TYPE_MAP = {"STRING": "literal", "WILDCARD": "wildcard", "REGEX": "regex"} + +PRISMA_ACTION_MAP = {'block': 'deny', 'allow': 'allow', 'monitor': 'alert'} +PRISMA_CATEGORY_MAP = {'malware': 'malware', 'phishing': 'phishing', 'gambling': 'gambling', 'uncategorized': 'uncategorized'} +PRISMA_TYPE_MAP = {"simple": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} + +# ---------------- PIPELINE DEFINITIONS ---------------- + +VENDOR_TO_UNIVERSAL_PIPELINES = { + 'fortinet': [ + ActionMapper(FORTINET_ACTION_MAP), + PatternNormalizer(), + TypeMapper(FORTINET_TYPE_MAP), + CategoryMapper(FORTINET_CATEGORY_MAP), + MetadataEnricher('fortinet') + ], + 'netskope': [ + ActionMapper(NETSKOPE_ACTION_MAP), + PatternNormalizer(), + TypeMapper(NETSKOPE_TYPE_MAP), + CategoryMapper(NETSKOPE_CATEGORY_MAP), + MetadataEnricher('netskope') + ], + 'zscaler': [ + ActionMapper(ZSCALER_ACTION_MAP), + PatternNormalizer(), + TypeMapper(ZSCALER_TYPE_MAP), + CategoryMapper(ZSCALER_CATEGORY_MAP), + MetadataEnricher('zscaler') + ], + 'prisma': [ + ActionMapper(PRISMA_ACTION_MAP), + PatternNormalizer(), + TypeMapper(PRISMA_TYPE_MAP), + CategoryMapper(PRISMA_CATEGORY_MAP), + MetadataEnricher('prisma') + ] +} + +UNIVERSAL_TO_VENDOR_PIPELINES = { + 'fortinet': [ + ActionMapper({v:k for k,v in FORTINET_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v:k for k,v in FORTINET_TYPE_MAP.items()}), + CategoryMapper({v:k for k,v in FORTINET_CATEGORY_MAP.items()}), + MetadataEnricher('fortinet') + ], + 'netskope': [ + ActionMapper({v:k for k,v in NETSKOPE_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v:k for k,v in NETSKOPE_TYPE_MAP.items()}), + CategoryMapper({v:k for k,v in NETSKOPE_CATEGORY_MAP.items()}), + MetadataEnricher('netskope') + ], + 'zscaler': [ + ActionMapper({v:k for k,v in ZSCALER_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v:k for k,v in ZSCALER_TYPE_MAP.items()}), + CategoryMapper({v:k for k,v in ZSCALER_CATEGORY_MAP.items()}), + MetadataEnricher('zscaler') + ], + 'prisma': [ + ActionMapper({v:k for k,v in PRISMA_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v:k for k,v in PRISMA_TYPE_MAP.items()}), + CategoryMapper({v:k for k,v in PRISMA_CATEGORY_MAP.items()}), + MetadataEnricher('prisma') + ] +} + + +A +A +A +A +A +A +A +A +A +A +A +A +A +A +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +A +A +A +A +A +A +A +A +A +A +A +A +A +A +A +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B +B + From d322e622b295d5c2d8f90d1eaff7b78b0eec88dc Mon Sep 17 00:00:00 2001 From: Elza Latifi Date: Tue, 2 Dec 2025 14:08:07 +0100 Subject: [PATCH 2/9] Clean up whitespace and empty lines in transformers.py Removed excessive whitespace and empty lines from the file. --- transformers/transformers.py | 87 ------------------------------------ 1 file changed, 87 deletions(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index 6081d480..fc4309b7 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -169,90 +169,3 @@ def apply_transformers(items: List[Dict[str, Any]], transformers: List[BaseTrans MetadataEnricher('prisma') ] } - - -A -A -A -A -A -A -A -A -A -A -A -A -A -A -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -A -A -A -A -A -A -A -A -A -A -A -A -A -A -A -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B -B - From 52c0e1df1a368ce6a68fea19c7bafffedd58f7f4 Mon Sep 17 00:00:00 2001 From: Elza Latifi Date: Wed, 3 Dec 2025 15:24:47 +0100 Subject: [PATCH 3/9] Refactor transformers.py for consistency and clarity --- transformers/transformers.py | 185 +++++++++++++++++++++++++---------- 1 file changed, 132 insertions(+), 53 deletions(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index fc4309b7..2925af42 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -15,72 +15,115 @@ # ---------------- UNIVERSAL DATA MODEL ---------------- + @dataclass class UniversalURLFilter: + """ """ + pattern: str action: str category: str + list_name: str = "" + list_id: str = "" type: str = "literal" # literal, wildcard, regex, substring vendor: str = "" metadata: Dict[str, Any] = field(default_factory=dict) + # ---------------- BASIC TRANSFORMERS ---------------- + class BaseTransformer: """Abstract transformer""" + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: raise NotImplementedError + class ActionMapper(BaseTransformer): + """ """ + def __init__(self, action_map: Dict[str, str]): self.action_map = action_map + + """ """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - item['action'] = self.action_map.get(item.get('action'), item.get('action')) + item["action"] = self.action_map.get(item.get("action"), item.get("action")) return item + class PatternNormalizer(BaseTransformer): """Keep pattern as-is; do not change type""" + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - item['pattern'] = item.get('pattern', '') + item["pattern"] = item.get("pattern", "") return item + class TypeMapper(BaseTransformer): """Map vendor type -> universal type""" + def __init__(self, type_map: Dict[str, str]): self.type_map = type_map + """ """ + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: # Always read the original 'type' from vendor config - vendor_type = item.get('type', 'simple') + vendor_type = item.get("type", "simple") # Normalize case for Zscaler - vendor_type_normalized = vendor_type.upper() if vendor_type in ['STRING', 'WILDCARD', 'REGEX'] else vendor_type.lower() + vendor_type_normalized = ( + vendor_type.upper() + if vendor_type in ["STRING", "WILDCARD", "REGEX"] + else vendor_type.lower() + ) # Map to universal type - item['type'] = self.type_map.get(vendor_type_normalized, 'literal') + item["type"] = self.type_map.get(vendor_type_normalized, "literal") return item + class CategoryMapper(BaseTransformer): """Map vendor category to universal category""" + def __init__(self, category_map: Dict[str, str]): self.category_map = category_map + """ """ + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - vendor_cat = str(item.get('category_id', 'default')) # read from input - item['category'] = self.category_map.get(vendor_cat, 'uncategorized') + vendor_cat = str(item.get("category_id", "default")) # read from input + item["category"] = self.category_map.get(vendor_cat, "uncategorized") return item + class MetadataEnricher(BaseTransformer): """Add vendor metadata""" - def __init__(self, vendor: str): + + def __init__(self, vendor: str, extra_fields: list = None): self.vendor = vendor + self.extra_fields = extra_fields or [] + + """ """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - item['vendor'] = self.vendor + item["vendor"] = self.vendor + if "metadata" not in item: + item["metadata"] = {} + + for field_name in self.extra_fields: + if field_name in item: + item["metadata"][field_name] = item[field_name] + return item + # ---------------- PIPELINE EXECUTOR ---------------- +""" """ -def apply_transformers(items: List[Dict[str, Any]], transformers: List[BaseTransformer]) -> List[Dict[str, Any]]: +def apply_transformers( + items: List[Dict[str, Any]], transformers: List[BaseTransformer] +) -> List[Dict[str, Any]]: result = [] for item in items: for t in transformers: @@ -88,84 +131,120 @@ def apply_transformers(items: List[Dict[str, Any]], transformers: List[BaseTrans result.append(item) return result -# ---------------- VENDOR MAPPINGS ---------------- -FORTINET_ACTION_MAP = {'block': 'block', 'allow': 'allow', 'monitor': 'monitor'} -FORTINET_CATEGORY_MAP = {'3': 'malware', '4': 'phishing', '5': 'gambling', 'default': 'uncategorized'} -FORTINET_TYPE_MAP = { "simple": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} +# ---------------- VENDOR MAPPINGS ---------------- -NETSKOPE_ACTION_MAP = {'block': 'deny', 'allow': 'allow', 'monitor': 'monitor'} -NETSKOPE_CATEGORY_MAP = {'malware': 'malware', 'phishing': 'phishing', 'gambling': 'gambling', 'uncategorized': 'uncategorized'} -NETSKOPE_TYPE_MAP = { "exact": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} +FORTINET_ACTION_MAP = {"block": "block", "allow": "allow", "monitor": "monitor"} +FORTINET_CATEGORY_MAP = { + "3": "malware", + "4": "phishing", + "5": "gambling", + "default": "uncategorized", +} +FORTINET_TYPE_MAP = { + "simple": "literal", + "wildcard": "wildcard", + "regex": "regex", + "substring": "substring", +} -ZSCALER_ACTION_MAP = {'block': 'BLOCK', 'allow': 'ALLOW', 'monitor': 'MONITOR'} -ZSCALER_CATEGORY_MAP = {'malware': 'malware', 'phishing': 'phishing', 'gambling': 'gambling', 'uncategorized': 'uncategorized'} -ZSCALER_TYPE_MAP = {"STRING": "literal", "WILDCARD": "wildcard", "REGEX": "regex"} +NETSKOPE_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "monitor"} +NETSKOPE_CATEGORY_MAP = { + "malware": "malware", + "phishing": "phishing", + "gambling": "gambling", + "uncategorized": "uncategorized", +} +NETSKOPE_TYPE_MAP = { + "exact": "literal", + "wildcard": "wildcard", + "regex": "regex", + "substring": "substring", +} -PRISMA_ACTION_MAP = {'block': 'deny', 'allow': 'allow', 'monitor': 'alert'} -PRISMA_CATEGORY_MAP = {'malware': 'malware', 'phishing': 'phishing', 'gambling': 'gambling', 'uncategorized': 'uncategorized'} -PRISMA_TYPE_MAP = {"simple": "literal", "wildcard": "wildcard", "regex": "regex", "substring": "substring"} +ZSCALER_ACTION_MAP = {"block": "BLOCK", "allow": "ALLOW", "monitor": "MONITOR"} +ZSCALER_CATEGORY_MAP = { + "malware": "malware", + "phishing": "phishing", + "gambling": "gambling", + "uncategorized": "uncategorized", +} +ZSCALER_TYPE_MAP = {"STRING": "literal", "WILDCARD": "wildcard", "REGEX": "regex"} + +PRISMA_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "alert"} +PRISMA_CATEGORY_MAP = { + "malware": "malware", + "phishing": "phishing", + "gambling": "gambling", + "uncategorized": "uncategorized", +} +PRISMA_TYPE_MAP = { + "simple": "literal", + "wildcard": "wildcard", + "regex": "regex", + "substring": "substring", +} # ---------------- PIPELINE DEFINITIONS ---------------- VENDOR_TO_UNIVERSAL_PIPELINES = { - 'fortinet': [ + "fortinet": [ ActionMapper(FORTINET_ACTION_MAP), PatternNormalizer(), TypeMapper(FORTINET_TYPE_MAP), CategoryMapper(FORTINET_CATEGORY_MAP), - MetadataEnricher('fortinet') + MetadataEnricher("fortinet"), ], - 'netskope': [ + "netskope": [ ActionMapper(NETSKOPE_ACTION_MAP), PatternNormalizer(), TypeMapper(NETSKOPE_TYPE_MAP), CategoryMapper(NETSKOPE_CATEGORY_MAP), - MetadataEnricher('netskope') + MetadataEnricher("netskope"), ], - 'zscaler': [ + "zscaler": [ ActionMapper(ZSCALER_ACTION_MAP), PatternNormalizer(), TypeMapper(ZSCALER_TYPE_MAP), CategoryMapper(ZSCALER_CATEGORY_MAP), - MetadataEnricher('zscaler') + MetadataEnricher("zscaler"), ], - 'prisma': [ + "prisma": [ ActionMapper(PRISMA_ACTION_MAP), PatternNormalizer(), TypeMapper(PRISMA_TYPE_MAP), CategoryMapper(PRISMA_CATEGORY_MAP), - MetadataEnricher('prisma') - ] + MetadataEnricher("prisma"), + ], } UNIVERSAL_TO_VENDOR_PIPELINES = { - 'fortinet': [ - ActionMapper({v:k for k,v in FORTINET_ACTION_MAP.items()}), + "fortinet": [ + ActionMapper({v: k for k, v in FORTINET_ACTION_MAP.items()}), PatternNormalizer(), - TypeMapper({v:k for k,v in FORTINET_TYPE_MAP.items()}), - CategoryMapper({v:k for k,v in FORTINET_CATEGORY_MAP.items()}), - MetadataEnricher('fortinet') + TypeMapper({v: k for k, v in FORTINET_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in FORTINET_CATEGORY_MAP.items()}), + MetadataEnricher("fortinet"), ], - 'netskope': [ - ActionMapper({v:k for k,v in NETSKOPE_ACTION_MAP.items()}), + "netskope": [ + ActionMapper({v: k for k, v in NETSKOPE_ACTION_MAP.items()}), PatternNormalizer(), - TypeMapper({v:k for k,v in NETSKOPE_TYPE_MAP.items()}), - CategoryMapper({v:k for k,v in NETSKOPE_CATEGORY_MAP.items()}), - MetadataEnricher('netskope') + TypeMapper({v: k for k, v in NETSKOPE_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in NETSKOPE_CATEGORY_MAP.items()}), + MetadataEnricher("netskope"), ], - 'zscaler': [ - ActionMapper({v:k for k,v in ZSCALER_ACTION_MAP.items()}), + "zscaler": [ + ActionMapper({v: k for k, v in ZSCALER_ACTION_MAP.items()}), PatternNormalizer(), - TypeMapper({v:k for k,v in ZSCALER_TYPE_MAP.items()}), - CategoryMapper({v:k for k,v in ZSCALER_CATEGORY_MAP.items()}), - MetadataEnricher('zscaler') + TypeMapper({v: k for k, v in ZSCALER_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in ZSCALER_CATEGORY_MAP.items()}), + MetadataEnricher("zscaler"), ], - 'prisma': [ - ActionMapper({v:k for k,v in PRISMA_ACTION_MAP.items()}), + "prisma": [ + ActionMapper({v: k for k, v in PRISMA_ACTION_MAP.items()}), PatternNormalizer(), - TypeMapper({v:k for k,v in PRISMA_TYPE_MAP.items()}), - CategoryMapper({v:k for k,v in PRISMA_CATEGORY_MAP.items()}), - MetadataEnricher('prisma') - ] + TypeMapper({v: k for k, v in PRISMA_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in PRISMA_CATEGORY_MAP.items()}), + MetadataEnricher("prisma"), + ], } From 926a36a1768760d8dd683454e6172582c4346be8 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Wed, 3 Dec 2025 15:31:23 +0100 Subject: [PATCH 4/9] Update transformers.py --- transformers/transformers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index 2925af42..ac15ddb8 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -18,7 +18,7 @@ @dataclass class UniversalURLFilter: - """ """ + """Universal URL Filter Model""" pattern: str action: str @@ -41,12 +41,12 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: class ActionMapper(BaseTransformer): - """ """ + """Action Mapping Actionner""" def __init__(self, action_map: Dict[str, str]): self.action_map = action_map - """ """ + """Action Mapping transformer""" def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: item["action"] = self.action_map.get(item.get("action"), item.get("action")) @@ -67,7 +67,7 @@ class TypeMapper(BaseTransformer): def __init__(self, type_map: Dict[str, str]): self.type_map = type_map - """ """ + """Type Mapping transformer """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: # Always read the original 'type' from vendor config @@ -89,7 +89,7 @@ class CategoryMapper(BaseTransformer): def __init__(self, category_map: Dict[str, str]): self.category_map = category_map - """ """ + """Category Mapping transformer """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: vendor_cat = str(item.get("category_id", "default")) # read from input @@ -104,7 +104,7 @@ def __init__(self, vendor: str, extra_fields: list = None): self.vendor = vendor self.extra_fields = extra_fields or [] - """ """ + """Metadata transformer """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: item["vendor"] = self.vendor @@ -119,7 +119,7 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: # ---------------- PIPELINE EXECUTOR ---------------- -""" """ +"""Apply transformers""" def apply_transformers( items: List[Dict[str, Any]], transformers: List[BaseTransformer] From 1290f1412e0bbd621e537c36e6144c892a0cc1d7 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Wed, 3 Dec 2025 15:40:33 +0100 Subject: [PATCH 5/9] Update transformers.py --- transformers/transformers.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index ac15ddb8..cad0500e 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -37,6 +37,7 @@ class BaseTransformer: """Abstract transformer""" def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Base transformer""" raise NotImplementedError @@ -44,11 +45,12 @@ class ActionMapper(BaseTransformer): """Action Mapping Actionner""" def __init__(self, action_map: Dict[str, str]): + """Init method""" self.action_map = action_map - - """Action Mapping transformer""" def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Action Mapping transformer""" + item["action"] = self.action_map.get(item.get("action"), item.get("action")) return item @@ -65,11 +67,13 @@ class TypeMapper(BaseTransformer): """Map vendor type -> universal type""" def __init__(self, type_map: Dict[str, str]): + """Init method""" + self.type_map = type_map - """Type Mapping transformer """ - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Type Mapping transformer """ + # Always read the original 'type' from vendor config vendor_type = item.get("type", "simple") # Normalize case for Zscaler @@ -87,12 +91,14 @@ class CategoryMapper(BaseTransformer): """Map vendor category to universal category""" def __init__(self, category_map: Dict[str, str]): + """Init method""" + self.category_map = category_map """Category Mapping transformer """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - vendor_cat = str(item.get("category_id", "default")) # read from input + vendor_cat = str(item.get("category_id", "default")) item["category"] = self.category_map.get(vendor_cat, "uncategorized") return item @@ -101,12 +107,14 @@ class MetadataEnricher(BaseTransformer): """Add vendor metadata""" def __init__(self, vendor: str, extra_fields: list = None): + """Init method""" + self.vendor = vendor self.extra_fields = extra_fields or [] - """Metadata transformer """ - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """Metadata transformer """ + item["vendor"] = self.vendor if "metadata" not in item: item["metadata"] = {} @@ -119,11 +127,12 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: # ---------------- PIPELINE EXECUTOR ---------------- -"""Apply transformers""" def apply_transformers( items: List[Dict[str, Any]], transformers: List[BaseTransformer] ) -> List[Dict[str, Any]]: + """Apply transformers""" + result = [] for item in items: for t in transformers: From c63947f2112c37fd5f578403a3a47c37736c1631 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Wed, 3 Dec 2025 15:46:29 +0100 Subject: [PATCH 6/9] Update transformers.py --- transformers/transformers.py | 289 ++++++++++++----------------------- 1 file changed, 97 insertions(+), 192 deletions(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index cad0500e..ed111e40 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -1,31 +1,44 @@ """ -transformers.py -A single-file SDK for bidirectional URL-filter migration transformers. - -Supports: -- Vendors: Fortinet, Netskope, Zscaler, Prisma Access -- Universal intermediate model -- Modular basic transformers (action, pattern, category, metadata) -- CLI and unit test support -- Built-in sample data for direct testing +Single-file SDK for bidirectional URL-filter migration transformers. + +This module provides: +- Universal intermediate URL-filter model +- Vendor-specific transformers for: Fortinet, Netskope, Zscaler, Prisma Access +- Modular basic transformers (action, pattern, type, category, metadata) +- Transformer pipelines (vendor → universal and universal → vendor) +- CLI and unit-test support +- Built-in sample mappings for immediate use """ from typing import List, Dict, Any from dataclasses import dataclass, field + # ---------------- UNIVERSAL DATA MODEL ---------------- @dataclass class UniversalURLFilter: - """Universal URL Filter Model""" + """ + Universal URL Filter Model. + + Attributes: + pattern: URL pattern from vendor configuration. + action: Universal action (allow, block, monitor, etc.). + category: Universal category name. + list_name: Optional list name. + list_id: Optional list identifier. + type: Pattern type (literal, wildcard, regex, substring). + vendor: Originating vendor name. + metadata: Additional vendor-specific fields. + """ pattern: str action: str category: str list_name: str = "" list_id: str = "" - type: str = "literal" # literal, wildcard, regex, substring + type: str = "literal" vendor: str = "" metadata: Dict[str, Any] = field(default_factory=dict) @@ -34,226 +47,118 @@ class UniversalURLFilter: class BaseTransformer: - """Abstract transformer""" + """ + Abstract transformer base class. + + All transformers must implement the ``transform`` method. + """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Base transformer""" + """ + Transform a single item. + + Args: + item: Input dictionary from vendor configuration. + + Returns: + Transformed dictionary. + + Raises: + NotImplementedError: If the subclass does not override this method. + """ raise NotImplementedError class ActionMapper(BaseTransformer): - """Action Mapping Actionner""" + """ + Transformer to map vendor actions to universal actions (or vice versa). + """ def __init__(self, action_map: Dict[str, str]): - """Init method""" + """ + Initialize the ActionMapper. + + Args: + action_map: Dictionary mapping vendor actions to universal actions. + """ self.action_map = action_map def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Action Mapping transformer""" - + """ + Map the action field using the configured mapping. + + Args: + item: Input dictionary to transform. + + Returns: + Updated dictionary with mapped action. + """ item["action"] = self.action_map.get(item.get("action"), item.get("action")) return item class PatternNormalizer(BaseTransformer): - """Keep pattern as-is; do not change type""" + """ + Transformer that normalizes or passes through patterns unchanged. + """ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """ + Normalize the pattern of the item. + + Args: + item: Input dictionary. + + Returns: + Updated dictionary with normalized pattern. + """ item["pattern"] = item.get("pattern", "") return item class TypeMapper(BaseTransformer): - """Map vendor type -> universal type""" + """ + Transformer that maps vendor pattern types to universal types (or reverse). + """ def __init__(self, type_map: Dict[str, str]): - """Init method""" - + """ + Initialize the TypeMapper. + + Args: + type_map: Mapping from vendor type to universal type. + """ self.type_map = type_map def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Type Mapping transformer """ - - # Always read the original 'type' from vendor config + """ + Map the pattern type, normalizing case when necessary. + + Args: + item: Input dictionary. + + Returns: + Updated dictionary with normalized pattern type. + """ vendor_type = item.get("type", "simple") - # Normalize case for Zscaler + vendor_type_normalized = ( vendor_type.upper() if vendor_type in ["STRING", "WILDCARD", "REGEX"] else vendor_type.lower() ) - # Map to universal type + item["type"] = self.type_map.get(vendor_type_normalized, "literal") return item class CategoryMapper(BaseTransformer): - """Map vendor category to universal category""" + """ + Transformer to map vendor category IDs or names to universal names. + """ def __init__(self, category_map: Dict[str, str]): - """Init method""" - - self.category_map = category_map - - """Category Mapping transformer """ - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - vendor_cat = str(item.get("category_id", "default")) - item["category"] = self.category_map.get(vendor_cat, "uncategorized") - return item - - -class MetadataEnricher(BaseTransformer): - """Add vendor metadata""" - - def __init__(self, vendor: str, extra_fields: list = None): - """Init method""" - - self.vendor = vendor - self.extra_fields = extra_fields or [] - - def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: - """Metadata transformer """ - - item["vendor"] = self.vendor - if "metadata" not in item: - item["metadata"] = {} - - for field_name in self.extra_fields: - if field_name in item: - item["metadata"][field_name] = item[field_name] - - return item - + """ + Initialize the CategoryMapper. -# ---------------- PIPELINE EXECUTOR ---------------- - -def apply_transformers( - items: List[Dict[str, Any]], transformers: List[BaseTransformer] -) -> List[Dict[str, Any]]: - """Apply transformers""" - - result = [] - for item in items: - for t in transformers: - item = t.transform(item) - result.append(item) - return result - - -# ---------------- VENDOR MAPPINGS ---------------- - -FORTINET_ACTION_MAP = {"block": "block", "allow": "allow", "monitor": "monitor"} -FORTINET_CATEGORY_MAP = { - "3": "malware", - "4": "phishing", - "5": "gambling", - "default": "uncategorized", -} -FORTINET_TYPE_MAP = { - "simple": "literal", - "wildcard": "wildcard", - "regex": "regex", - "substring": "substring", -} - -NETSKOPE_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "monitor"} -NETSKOPE_CATEGORY_MAP = { - "malware": "malware", - "phishing": "phishing", - "gambling": "gambling", - "uncategorized": "uncategorized", -} -NETSKOPE_TYPE_MAP = { - "exact": "literal", - "wildcard": "wildcard", - "regex": "regex", - "substring": "substring", -} - -ZSCALER_ACTION_MAP = {"block": "BLOCK", "allow": "ALLOW", "monitor": "MONITOR"} -ZSCALER_CATEGORY_MAP = { - "malware": "malware", - "phishing": "phishing", - "gambling": "gambling", - "uncategorized": "uncategorized", -} -ZSCALER_TYPE_MAP = {"STRING": "literal", "WILDCARD": "wildcard", "REGEX": "regex"} - -PRISMA_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "alert"} -PRISMA_CATEGORY_MAP = { - "malware": "malware", - "phishing": "phishing", - "gambling": "gambling", - "uncategorized": "uncategorized", -} -PRISMA_TYPE_MAP = { - "simple": "literal", - "wildcard": "wildcard", - "regex": "regex", - "substring": "substring", -} - -# ---------------- PIPELINE DEFINITIONS ---------------- - -VENDOR_TO_UNIVERSAL_PIPELINES = { - "fortinet": [ - ActionMapper(FORTINET_ACTION_MAP), - PatternNormalizer(), - TypeMapper(FORTINET_TYPE_MAP), - CategoryMapper(FORTINET_CATEGORY_MAP), - MetadataEnricher("fortinet"), - ], - "netskope": [ - ActionMapper(NETSKOPE_ACTION_MAP), - PatternNormalizer(), - TypeMapper(NETSKOPE_TYPE_MAP), - CategoryMapper(NETSKOPE_CATEGORY_MAP), - MetadataEnricher("netskope"), - ], - "zscaler": [ - ActionMapper(ZSCALER_ACTION_MAP), - PatternNormalizer(), - TypeMapper(ZSCALER_TYPE_MAP), - CategoryMapper(ZSCALER_CATEGORY_MAP), - MetadataEnricher("zscaler"), - ], - "prisma": [ - ActionMapper(PRISMA_ACTION_MAP), - PatternNormalizer(), - TypeMapper(PRISMA_TYPE_MAP), - CategoryMapper(PRISMA_CATEGORY_MAP), - MetadataEnricher("prisma"), - ], -} - -UNIVERSAL_TO_VENDOR_PIPELINES = { - "fortinet": [ - ActionMapper({v: k for k, v in FORTINET_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({v: k for k, v in FORTINET_TYPE_MAP.items()}), - CategoryMapper({v: k for k, v in FORTINET_CATEGORY_MAP.items()}), - MetadataEnricher("fortinet"), - ], - "netskope": [ - ActionMapper({v: k for k, v in NETSKOPE_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({v: k for k, v in NETSKOPE_TYPE_MAP.items()}), - CategoryMapper({v: k for k, v in NETSKOPE_CATEGORY_MAP.items()}), - MetadataEnricher("netskope"), - ], - "zscaler": [ - ActionMapper({v: k for k, v in ZSCALER_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({v: k for k, v in ZSCALER_TYPE_MAP.items()}), - CategoryMapper({v: k for k, v in ZSCALER_CATEGORY_MAP.items()}), - MetadataEnricher("zscaler"), - ], - "prisma": [ - ActionMapper({v: k for k, v in PRISMA_ACTION_MAP.items()}), - PatternNormalizer(), - TypeMapper({v: k for k, v in PRISMA_TYPE_MAP.items()}), - CategoryMapper({v: k for k, v in PRISMA_CATEGORY_MAP.items()}), - MetadataEnricher("prisma"), - ], -} + A From 4ee6dd0338257af783e7e9918a70564331766c63 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Wed, 3 Dec 2025 15:50:37 +0100 Subject: [PATCH 7/9] Update transformers.py --- transformers/transformers.py | 199 ++++++++++++++++++++++++++++++++++- 1 file changed, 198 insertions(+), 1 deletion(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index ed111e40..9555e23d 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -161,4 +161,201 @@ def __init__(self, category_map: Dict[str, str]): """ Initialize the CategoryMapper. - A + Args: + category_map: Mapping from vendor category to universal category. + """ + self.category_map = category_map + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """ + Map vendor category to a universal category. + + Args: + item: Input dictionary. + + Returns: + Updated dictionary with normalized category. + """ + vendor_cat = str(item.get("category_id", "default")) + item["category"] = self.category_map.get(vendor_cat, "uncategorized") + return item + + +class MetadataEnricher(BaseTransformer): + """ + Transformer that enriches items with vendor metadata fields. + """ + + def __init__(self, vendor: str, extra_fields: List[str] | None = None): + """ + Initialize the MetadataEnricher. + + Args: + vendor: Vendor identifier. + extra_fields: Optional list of field names to embed in metadata. + """ + self.vendor = vendor + self.extra_fields = extra_fields or [] + + def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: + """ + Add vendor metadata to the item. + + Args: + item: Input dictionary. + + Returns: + Updated dictionary enriched with metadata. + """ + item["vendor"] = self.vendor + item.setdefault("metadata", {}) + + for field_name in self.extra_fields: + if field_name in item: + item["metadata"][field_name] = item[field_name] + + return item + + +# ---------------- PIPELINE EXECUTOR ---------------- + + +def apply_transformers( + items: List[Dict[str, Any]], transformers: List[BaseTransformer] +) -> List[Dict[str, Any]]: + """ + Apply a list of transformers sequentially to a list of items. + + Args: + items: List of vendor configuration dictionaries. + transformers: Ordered list of transformer instances. + + Returns: + A list of transformed dictionaries. + """ + result = [] + for item in items: + for transformer in transformers: + item = transformer.transform(item) + result.append(item) + return result + + +# ---------------- VENDOR MAPPINGS ---------------- +# (constant values do not require docstrings) + +FORTINET_ACTION_MAP = {"block": "block", "allow": "allow", "monitor": "monitor"} +FORTINET_CATEGORY_MAP = { + "3": "malware", + "4": "phishing", + "5": "gambling", + "default": "uncategorized", +} +FORTINET_TYPE_MAP = { + "simple": "literal", + "wildcard": "wildcard", + "regex": "regex", + "substring": "substring", +} + +NETSKOPE_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "monitor"} +NETSKOPE_CATEGORY_MAP = { + "malware": "malware", + "phishing": "phishing", + "gambling": "gambling", + "uncategorized": "uncategorized", +} +NETSKOPE_TYPE_MAP = { + "exact": "literal", + "wildcard": "wildcard", + "regex": "regex", + "substring": "substring", +} + +ZSCALER_ACTION_MAP = {"block": "BLOCK", "allow": "ALLOW", "monitor": "MONITOR"} +ZSCALER_CATEGORY_MAP = { + "malware": "malware", + "phishing": "phishing", + "gambling": "gambling", + "uncategorized": "uncategorized", +} +ZSCALER_TYPE_MAP = {"STRING": "literal", "WILDCARD": "wildcard", "REGEX": "regex"} + +PRISMA_ACTION_MAP = {"block": "deny", "allow": "allow", "monitor": "alert"} +PRISMA_CATEGORY_MAP = { + "malware": "malware", + "phishing": "phishing", + "gambling": "gambling", + "uncategorized": "uncategorized", +} +PRISMA_TYPE_MAP = { + "simple": "literal", + "wildcard": "wildcard", + "regex": "regex", + "substring": "substring", +} + + +# ---------------- PIPELINE DEFINITIONS ---------------- + +VENDOR_TO_UNIVERSAL_PIPELINES = { + "fortinet": [ + ActionMapper(FORTINET_ACTION_MAP), + PatternNormalizer(), + TypeMapper(FORTINET_TYPE_MAP), + CategoryMapper(FORTINET_CATEGORY_MAP), + MetadataEnricher("fortinet"), + ], + "netskope": [ + ActionMapper(NETSKOPE_ACTION_MAP), + PatternNormalizer(), + TypeMapper(NETSKOPE_TYPE_MAP), + CategoryMapper(NETSKOPE_CATEGORY_MAP), + MetadataEnricher("netskope"), + ], + "zscaler": [ + ActionMapper(ZSCALER_ACTION_MAP), + PatternNormalizer(), + TypeMapper(ZSCALER_TYPE_MAP), + CategoryMapper(ZSCALER_CATEGORY_MAP), + MetadataEnricher("zscaler"), + ], + "prisma": [ + ActionMapper(PRISMA_ACTION_MAP), + PatternNormalizer(), + TypeMapper(PRISMA_TYPE_MAP), + CategoryMapper(PRISMA_CATEGORY_MAP), + MetadataEnricher("prisma"), + ], +} + +UNIVERSAL_TO_VENDOR_PIPELINES = { + "fortinet": [ + ActionMapper({v: k for k, v in FORTINET_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v: k for k, v in FORTINET_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in FORTINET_CATEGORY_MAP.items()}), + MetadataEnricher("fortinet"), + ], + "netskope": [ + ActionMapper({v: k for k, v in NETSKOPE_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v: k for k, v in NETSKOPE_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in NETSKOPE_CATEGORY_MAP.items()}), + MetadataEnricher("netskope"), + ], + "zscaler": [ + ActionMapper({v: k for k, v in ZSCALER_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v: k for k, v in ZSCALER_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in ZSCALER_CATEGORY_MAP.items()}), + MetadataEnricher("zscaler"), + ], + "prisma": [ + ActionMapper({v: k for k, v in PRISMA_ACTION_MAP.items()}), + PatternNormalizer(), + TypeMapper({v: k for k, v in PRISMA_TYPE_MAP.items()}), + CategoryMapper({v: k for k, v in PRISMA_CATEGORY_MAP.items()}), + MetadataEnricher("prisma"), + ], +} From 93bc00d263d1cc0572c33ab33e85e16cc5d84fe3 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Wed, 3 Dec 2025 15:56:09 +0100 Subject: [PATCH 8/9] Update transformers.py --- transformers/transformers.py | 68 ++++++++++++------------------------ 1 file changed, 23 insertions(+), 45 deletions(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index 9555e23d..591694ef 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -19,19 +19,7 @@ @dataclass class UniversalURLFilter: - """ - Universal URL Filter Model. - - Attributes: - pattern: URL pattern from vendor configuration. - action: Universal action (allow, block, monitor, etc.). - category: Universal category name. - list_name: Optional list name. - list_id: Optional list identifier. - type: Pattern type (literal, wildcard, regex, substring). - vendor: Originating vendor name. - metadata: Additional vendor-specific fields. - """ + """Universal URL Filter Model.""" pattern: str action: str @@ -47,11 +35,7 @@ class UniversalURLFilter: class BaseTransformer: - """ - Abstract transformer base class. - - All transformers must implement the ``transform`` method. - """ + """Abstract base class for all transformers.""" def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: """ @@ -70,16 +54,14 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: class ActionMapper(BaseTransformer): - """ - Transformer to map vendor actions to universal actions (or vice versa). - """ + """Map vendor actions to universal actions (or reverse).""" def __init__(self, action_map: Dict[str, str]): """ Initialize the ActionMapper. Args: - action_map: Dictionary mapping vendor actions to universal actions. + action_map: Mapping from vendor action → universal action. """ self.action_map = action_map @@ -88,23 +70,21 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: Map the action field using the configured mapping. Args: - item: Input dictionary to transform. + item: Input dictionary. Returns: - Updated dictionary with mapped action. + Dictionary with updated action. """ item["action"] = self.action_map.get(item.get("action"), item.get("action")) return item class PatternNormalizer(BaseTransformer): - """ - Transformer that normalizes or passes through patterns unchanged. - """ + """Normalize or pass through patterns unchanged.""" def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: """ - Normalize the pattern of the item. + Normalize the pattern field. Args: item: Input dictionary. @@ -117,22 +97,23 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: class TypeMapper(BaseTransformer): - """ - Transformer that maps vendor pattern types to universal types (or reverse). - """ + """Map vendor pattern types to universal types (or reverse).""" def __init__(self, type_map: Dict[str, str]): """ Initialize the TypeMapper. Args: - type_map: Mapping from vendor type to universal type. + type_map: Mapping from vendor type → universal type. """ self.type_map = type_map def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: """ - Map the pattern type, normalizing case when necessary. + Map the pattern type field. + + Vendor types may be inconsistent (uppercase/lowercase), + so normalization rules apply before mapping. Args: item: Input dictionary. @@ -153,22 +134,20 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: class CategoryMapper(BaseTransformer): - """ - Transformer to map vendor category IDs or names to universal names. - """ + """Map vendor category IDs or names to universal names.""" def __init__(self, category_map: Dict[str, str]): """ Initialize the CategoryMapper. Args: - category_map: Mapping from vendor category to universal category. + category_map: Mapping from vendor category → universal. """ self.category_map = category_map def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: """ - Map vendor category to a universal category. + Map the category field. Args: item: Input dictionary. @@ -182,9 +161,7 @@ def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: class MetadataEnricher(BaseTransformer): - """ - Transformer that enriches items with vendor metadata fields. - """ + """Enrich items with vendor name and embedded metadata fields.""" def __init__(self, vendor: str, extra_fields: List[str] | None = None): """ @@ -192,14 +169,14 @@ def __init__(self, vendor: str, extra_fields: List[str] | None = None): Args: vendor: Vendor identifier. - extra_fields: Optional list of field names to embed in metadata. + extra_fields: Optional list of fields to copy to metadata. """ self.vendor = vendor self.extra_fields = extra_fields or [] def transform(self, item: Dict[str, Any]) -> Dict[str, Any]: """ - Add vendor metadata to the item. + Add vendor metadata fields to the item. Args: item: Input dictionary. @@ -231,7 +208,7 @@ def apply_transformers( transformers: Ordered list of transformer instances. Returns: - A list of transformed dictionaries. + List of transformed dictionaries. """ result = [] for item in items: @@ -242,7 +219,7 @@ def apply_transformers( # ---------------- VENDOR MAPPINGS ---------------- -# (constant values do not require docstrings) + FORTINET_ACTION_MAP = {"block": "block", "allow": "allow", "monitor": "monitor"} FORTINET_CATEGORY_MAP = { @@ -298,6 +275,7 @@ def apply_transformers( # ---------------- PIPELINE DEFINITIONS ---------------- + VENDOR_TO_UNIVERSAL_PIPELINES = { "fortinet": [ ActionMapper(FORTINET_ACTION_MAP), From f295d77276c34f98e40f4b7c69b586d4df6f6139 Mon Sep 17 00:00:00 2001 From: pierrerondel Date: Wed, 3 Dec 2025 16:00:33 +0100 Subject: [PATCH 9/9] Update transformers.py --- transformers/transformers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/transformers/transformers.py b/transformers/transformers.py index 591694ef..2914f535 100644 --- a/transformers/transformers.py +++ b/transformers/transformers.py @@ -10,9 +10,11 @@ - Built-in sample mappings for immediate use """ -from typing import List, Dict, Any -from dataclasses import dataclass, field - +from dataclasses import dataclass +from dataclasses import field +from typing import Any +from typing import Dict +from typing import List # ---------------- UNIVERSAL DATA MODEL ----------------