From 094d1162a6969a4375aa56c50b5b02d500122bc8 Mon Sep 17 00:00:00 2001 From: Nate Smalley Date: Sun, 26 Apr 2026 21:41:34 -0700 Subject: [PATCH 1/2] pipelines: drop 7 broken-legacy transform_ocsf entries Removes the following directories from pipelines/community/transform_ocsf/: aws_cloudtrail/ aws_guardduty/ darktrace/ gcp_audit_logs/ microsoft_365/ okta/ wiz_issue/ Each entry shares the same broken-legacy fingerprint (matching palo_alto_networks_firewall/ from #60): - Sub-passing grade (D or F). - verdict: analyzer_limit (the automated grader could not validate the serializer's OCSF output). - class_uid: null (no valid OCSF class is produced). - required_field_coverage_pct: 0. - source_name lacks the -latest versioning suffix used by every working entry in the directory. - No matching upstream parser in parsers/community/. - Long-form Python-port style code (632 to 1720 lines), imported from the Observo platform UI rather than via the standard contributor path. Each removed entry has at least one working alternative covering the same vendor cluster: aws_cloudtrail/ -> aws_*/transform_ocsf/ entries that bind to parsers/community/-latest/ (signed_off, B+ grade) aws_guardduty/ -> aws_guardduty_logs/ (B/85, signed_off, class_uid=2004) darktrace/ -> darktrace_darktrace_logs/ (B/85, signed_off, class_uid=2004) gcp_audit_logs/ -> use the bound-parser alternatives in the same vendor cluster microsoft_365/ -> microsoft_365_mgmt_api_logs/ (B/82, signed_off, class_uid=6003) okta/ -> okta_logs/ (B/85, signed_off, class_uid=3002) and okta_ocsf_logs/ (B/85, signed_off, class_uid=3002) wiz_issue/ -> wiz_cloud_security_logs/ (B/85, signed_off, class_uid=2004) No serializer logic, no other metadata, no pipeline JSON in the surviving entries was modified. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../aws_cloudtrail/aws_cloudtrail.json | 57 - .../aws_cloudtrail/metadata.yaml | 52 - .../transform_ocsf/aws_cloudtrail/sample.json | 89 - .../aws_cloudtrail/serializer.lua | 552 ------ .../aws_guardduty/aws_guardduty.json | 57 - .../aws_guardduty/metadata.yaml | 51 - .../transform_ocsf/aws_guardduty/sample.json | 55 - .../aws_guardduty/serializer.lua | 1720 ----------------- .../transform_ocsf/darktrace/darktrace.json | 57 - .../transform_ocsf/darktrace/metadata.yaml | 51 - .../transform_ocsf/darktrace/sample.json | 60 - .../transform_ocsf/darktrace/serializer.lua | 987 ---------- .../gcp_audit_logs/gcp_audit_logs.json | 57 - .../gcp_audit_logs/metadata.yaml | 47 - .../transform_ocsf/gcp_audit_logs/sample.json | 44 - .../gcp_audit_logs/serializer.lua | 623 ------ .../microsoft_365/metadata.yaml | 52 - .../microsoft_365/microsoft_365.json | 57 - .../transform_ocsf/microsoft_365/sample.json | 170 -- .../microsoft_365/serializer.lua | 1342 ------------- .../transform_ocsf/okta/metadata.yaml | 53 - .../community/transform_ocsf/okta/okta.json | 57 - .../community/transform_ocsf/okta/sample.json | 103 - .../transform_ocsf/okta/serializer.lua | 1495 -------------- .../transform_ocsf/wiz_issue/metadata.yaml | 52 - .../transform_ocsf/wiz_issue/sample.json | 67 - .../transform_ocsf/wiz_issue/serializer.lua | 632 ------ .../transform_ocsf/wiz_issue/wiz_issue.json | 57 - 28 files changed, 8696 deletions(-) delete mode 100644 pipelines/community/transform_ocsf/aws_cloudtrail/aws_cloudtrail.json delete mode 100644 pipelines/community/transform_ocsf/aws_cloudtrail/metadata.yaml delete mode 100644 pipelines/community/transform_ocsf/aws_cloudtrail/sample.json delete mode 100644 pipelines/community/transform_ocsf/aws_cloudtrail/serializer.lua delete mode 100644 pipelines/community/transform_ocsf/aws_guardduty/aws_guardduty.json delete mode 100644 pipelines/community/transform_ocsf/aws_guardduty/metadata.yaml delete mode 100644 pipelines/community/transform_ocsf/aws_guardduty/sample.json delete mode 100644 pipelines/community/transform_ocsf/aws_guardduty/serializer.lua delete mode 100644 pipelines/community/transform_ocsf/darktrace/darktrace.json delete mode 100644 pipelines/community/transform_ocsf/darktrace/metadata.yaml delete mode 100644 pipelines/community/transform_ocsf/darktrace/sample.json delete mode 100644 pipelines/community/transform_ocsf/darktrace/serializer.lua delete mode 100644 pipelines/community/transform_ocsf/gcp_audit_logs/gcp_audit_logs.json delete mode 100644 pipelines/community/transform_ocsf/gcp_audit_logs/metadata.yaml delete mode 100644 pipelines/community/transform_ocsf/gcp_audit_logs/sample.json delete mode 100644 pipelines/community/transform_ocsf/gcp_audit_logs/serializer.lua delete mode 100644 pipelines/community/transform_ocsf/microsoft_365/metadata.yaml delete mode 100644 pipelines/community/transform_ocsf/microsoft_365/microsoft_365.json delete mode 100644 pipelines/community/transform_ocsf/microsoft_365/sample.json delete mode 100644 pipelines/community/transform_ocsf/microsoft_365/serializer.lua delete mode 100644 pipelines/community/transform_ocsf/okta/metadata.yaml delete mode 100644 pipelines/community/transform_ocsf/okta/okta.json delete mode 100644 pipelines/community/transform_ocsf/okta/sample.json delete mode 100644 pipelines/community/transform_ocsf/okta/serializer.lua delete mode 100644 pipelines/community/transform_ocsf/wiz_issue/metadata.yaml delete mode 100644 pipelines/community/transform_ocsf/wiz_issue/sample.json delete mode 100644 pipelines/community/transform_ocsf/wiz_issue/serializer.lua delete mode 100644 pipelines/community/transform_ocsf/wiz_issue/wiz_issue.json diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/aws_cloudtrail.json b/pipelines/community/transform_ocsf/aws_cloudtrail/aws_cloudtrail.json deleted file mode 100644 index fea2497..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/aws_cloudtrail.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Aws Cloudtrail", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Aws Cloudtrail. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "aws", - "source_name": "aws_cloudtrail", - "version": "1.0.0-rc3", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "aws-cloudtrail-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "\nlocal FEATURES = {\n IGNORE_UNKNOWN_EVENT = true,\n}\n\n-- Field ordering templates for consistent JSON serialization\nlocal FIELD_ORDERS = {\n root = {\"eventVersion\", \"userIdentity\", \"eventTime\", \"eventSource\", \"eventName\", \"awsRegion\", \"sourceIPAddress\", \"userAgent\", \"requestParameters\", \"responseElements\", \"additionalEventData\", \"requestID\", \"eventID\", \"readOnly\", \"resources\", \"eventType\", \"managementEvent\", \"recipientAccountId\", \"sharedEventID\", \"eventCategory\", \"tlsDetails\", \"errorCode\", \"errorMessage\", \"vpcEndpointId\", \"apiVersion\", \"message\", \"time\", \"insightDetails\", \"class_uid\", \"category_uid\", \"metadata\", \"dataSource\"},\n userIdentity = {\"accountId\", \"principalId\", \"accessKeyId\", \"userName\", \"type\", \"invokedBy\", \"sessionContext\"},\n sessionContext = {\"sessionIssuer\", \"attributes\"},\n sessionIssuer = {\"type\", \"principalId\", \"arn\", \"accountId\", \"userName\"},\n attributes = {\"creationDate\"},\n requestParameters = {\"durationSeconds\", \"externalId\", \"bucketName\", \"Host\", \"instanceId\", \"availabilityZone\", \"requestContext\"},\n requestContext = {\"awsAccountId\"},\n responseElements = {\"credentials\"},\n credentials = {\"accessKeyId\", \"expiration\"},\n additionalEventData = {\"SignatureVersion\", \"CipherSuite\", \"bytesTransferredIn\", \"AuthenticationMethod\", \"x-amz-id-2\", \"bytesTransferredOut\"},\n resources = {\"accountId\", \"type\", \"ARN\"},\n tlsDetails = {\"tlsVersion\", \"cipherSuite\"},\n insightDetails = {\"eventSource\", \"insightContext\"},\n insightContext = {\"statistics\"},\n statistics = {\"insightDuration\"},\n metadata = {\"product\"},\n product = {\"name\", \"vendor_name\"},\n dataSource = {\"category\", \"vendor\", \"name\"}\n}\n\n-- Convert ISO 8601 timestamp to Unix epoch milliseconds\nfunction convertToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n \n -- Parse ISO 8601 format: \"2025-09-29T09:15:40Z\" or \"2025-09-29T09:15:40.123Z\"\n local year, month, day, hour, min, sec, ms = string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z\")\n \n if year and month and day and hour and min and sec then\n local t = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n \n -- Get local time interpretation\n local local_seconds = os.time(t)\n \n -- Get what this represents in UTC\n local utc_date = os.date(\"!*t\", local_seconds)\n \n local unix_seconds\n -- Check if the UTC interpretation matches our input\n if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and \n utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and \n utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then\n -- We are already in UTC, use as-is\n unix_seconds = local_seconds\n else\n -- Calculate the correct UTC timestamp\n local utc_seconds = os.time(utc_date)\n local offset = local_seconds - utc_seconds\n unix_seconds = local_seconds + offset -- Add offset to get UTC\n end\n \n -- Add milliseconds if present\n local milli = 0\n if ms and ms ~= \"\" then\n milli = tonumber((ms .. \"000\"):sub(1, 3)) -- pad/truncate to 3 digits\n end\n \n return unix_seconds * 1000 + milli\n end\n \n return nil\nend\n\n-- Optimized JSON encoding function with predefined ordering\nfunction encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" then\n return \"null\"\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n \n if isArray and maxIndex > 0 then\n local items = {}\n for i = 1, maxIndex do\n -- Use the parent key for predefined ordering if available\n local elementKey = key or tostring(i)\n table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or \"null\")\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n local items = {}\n local fieldOrder = FIELD_ORDERS[key] or {}\n \n -- Phase 1: Process fields in predefined order\n for _, fieldName in ipairs(fieldOrder) do\n local v = obj[fieldName]\n if v ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, fieldName))\n end\n end\n \n -- Phase 2: Process remaining fields\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then \n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, keyStr))\n end\n end\n \n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nfunction processEvent(event)\n \n -- Check if eventType is missing and IGNORE_UNKNOWN_EVENT is enabled\n if FEATURES.IGNORE_UNKNOWN_EVENT and (not event.eventType or event.eventType == \"\") then\n return nil\n end\n\n local result = {}\n \n -- Handle responseElements null case\n if event.responseElements == \"\" then\n event.responseElements = nil\n end\n \n -- Direct field mappings for better performance\n local function setField(sourcePath, targetPath)\n local value = getNestedField(event, sourcePath)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n end\n end\n \n -- Priority-based field mapping with fallback\n local function setFieldWithPriority(priority1, priority2, priority3, targetPath)\n local value = getNestedField(event, priority1)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n elseif priority2 then\n value = getNestedField(event, priority2)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n elseif priority3 and priority3 ~= \"\" then\n value = getNestedField(event, priority3)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n end\n end\n end\n end\n \n \n -- Field mapping table for better readability and maintainability\n local fieldMappings = {\n -- Basic CloudTrail fields\n {type = \"direct\", source = \"awsRegion\", target = \"cloud.region\"},\n {type = \"direct\", source = \"eventCategory\", target = \"metadata.product.feature.name\"},\n {type = \"direct\", source = \"eventID\", target = \"metadata.uid\"},\n {type = \"direct\", source = \"eventTime\", target = \"metadata.original_time\"},\n {type = \"direct\", source = \"eventVersion\", target = \"metadata.product.version\"},\n {type = \"direct\", source = \"recipientAccountId\", target = \"cloud.account.uid\"},\n {type = \"priority\", source1 = \"requestID\", source2 = \"requestParameters.externalId\", source3 = \"requestParameters.requestContext.awsAccountId\", target = \"api.request.uid\"},\n {type = \"direct\", source = \"sourceIPAddress\", target = \"src_endpoint.ip\"},\n {type = \"direct\", source = \"userAgent\", target = \"http_request.user_agent\"},\n \n -- User Identity fields\n {type = \"direct\", source = \"userIdentity.principalId\", target = \"actor.user.uid\"},\n {type = \"direct\", source = \"userIdentity.accessKeyId\", target = \"actor.user.credential_uid\"},\n {type = \"direct\", source = \"userIdentity.type\", target = \"actor.user.type\"},\n {type = \"direct\", source = \"userIdentity.invokedBy\", target = \"actor.invoked_by\"},\n {type = \"direct\", source = \"userIdentity.sessionContext.sessionIssuer.principalId\", target = \"actor.session.uid\"},\n {type = \"direct\", source = \"userIdentity.sessionContext.sessionIssuer.userName\", target = \"actor.session.issuer\"},\n {type = \"priority\", source1 = \"userIdentity.userName\", source2 = \"userIdentity.sessionContext.sessionIssuer.type\", target = \"actor.user.name\"},\n {type = \"priority\", source1 = \"userIdentity.accountId\", source2 = \"userIdentity.sessionContext.sessionIssuer.accountId\", target = \"actor.user.account.uid\"},\n \n -- API Information\n {type = \"direct\", source = \"errorCode\", target = \"api.response.error\"},\n {type = \"direct\", source = \"errorMessage\", target = \"api.response.error_message\"},\n \n -- Request Parameters\n {type = \"priority\", source1 = \"requestParameters.durationSeconds\", source2 = \"insightDetails.insightContext.statistics.insightDuration\", target = \"duration\"},\n {type = \"direct\", source = \"requestParameters.bucketName\", target = \"resources.name\"},\n {type = \"direct\", source = \"requestParameters.Host\", target = \"src_endpoint.hostname\"},\n {type = \"direct\", source = \"requestParameters.instanceId\", target = \"src_endpoint.instance_uid\"},\n {type = \"direct\", source = \"requestParameters.availabilityZone\", target = \"cloud.zone\"},\n \n -- Response Elements\n {type = \"direct\", source = \"responseElements.credentials.accessKeyId\", target = \"actor.session.credential_uid\"},\n {type = \"direct\", source = \"responseElements.credentials.expiration\", target = \"actor.session.expiration_time\"},\n \n -- Additional Event Data\n {type = \"direct\", source = \"additionalEventData.x-amz-id-2\", target = \"resources.uid\"},\n {type = \"direct\", source = \"tlsDetails.cipherSuite\", target = \"tls.cipher\"},\n {type = \"direct\", source = \"tlsDetails.tlsVersion\", target = \"tls.version\"},\n \n -- Additional fields\n {type = \"direct\", source = \"vpcEndpointId\", target = \"src_endpoint.uid\"},\n {type = \"direct\", source = \"apiVersion\", target = \"api.version\"},\n {type = \"direct\", source = \"message\", target = \"message\"},\n {type = \"priority\", source1 = \"eventSource\", source2 = \"insightDetails.eventSource\", target = \"api.service.name\"},\n \n -- Individual resource field mappings (for single resource events)\n {type = \"direct\", source = \"resources.accountId\", target = \"resource.account.uid\"},\n {type = \"direct\", source = \"resources.type\", target = \"resource.type\"},\n {type = \"direct\", source = \"resources.ARN\", target = \"resource.uid\"},\n\n -- OCSF field mappings (direct from input)\n {type = \"direct\", source = \"class_uid\", target = \"class_uid\"},\n {type = \"direct\", source = \"category_uid\", target = \"category_uid\"}\n }\n \n -- Process all field mappings in one iteration\n for _, mapping in ipairs(fieldMappings) do\n if mapping.type == \"direct\" then\n setField(mapping.source, mapping.target)\n elseif mapping.type == \"priority\" then\n setFieldWithPriority(mapping.source1, mapping.source2, mapping.source3, mapping.target)\n end\n end\n \n -- Convert timestamps to milliseconds\n local creationDate = getNestedField(event, 'userIdentity.sessionContext.attributes.creationDate')\n if creationDate then\n local convertedTime = convertToMilliseconds(creationDate)\n if convertedTime then\n setNestedField(result, 'actor.session.created_time', convertedTime)\n end\n end\n \n local expirationDate = getNestedField(event, 'responseElements.credentials.expiration')\n if expirationDate then\n local convertedTime = convertToMilliseconds(expirationDate)\n if convertedTime then\n setNestedField(result, 'actor.session.expiration_time', convertedTime)\n end\n end\n \n -- Convert eventTime to time field\n local eventTime = getNestedField(event, 'eventTime')\n if eventTime then\n local convertedTime = convertToMilliseconds(eventTime)\n if convertedTime then\n -- we need to use milliseconds here to be compatible with S1\n setNestedField(result, 'time', convertedTime)\n end\n end\n \n -- Resources array handling\n if event.resources and type(event.resources) == \"table\" then\n local accountIds = {}\n local types = {}\n local arns = {}\n \n for _, resource in ipairs(event.resources) do\n if resource.accountId then table.insert(accountIds, resource.accountId) end\n if resource.type then table.insert(types, resource.type) end\n if resource.ARN then table.insert(arns, resource.ARN) end\n end\n \n if #accountIds > 0 then setNestedField(result, 'resource.account.uid', accountIds) end\n if #types > 0 then setNestedField(result, 'resource.type', types) end\n if #arns > 0 then setNestedField(result, 'resource.uid', arns) end\n end\n \n -- Priority-based default OCSF fields (only set if not already present)\n local function setDefaultIfNotExists(targetPath, defaultValue)\n local existingValue = getNestedField(result, targetPath)\n if existingValue == nil then\n setNestedField(result, targetPath, defaultValue)\n end\n end\n\n -- Static category mapping\n setDefaultIfNotExists('category_uid', 4)\n \n -- Static class mapping \n setDefaultIfNotExists('class_uid', 4002)\n \n -- Set defaults only if fields don't already exist\n setDefaultIfNotExists('metadata.product.name', 'CloudTrail')\n setDefaultIfNotExists('metadata.product.vendor_name', 'AWS')\n setDefaultIfNotExists('metadata.version', '1.0.0-rc3')\n setDefaultIfNotExists('dataSource.vendor', 'AWS')\n setDefaultIfNotExists('dataSource.name', 'CloudTrail')\n setDefaultIfNotExists('dataSource.category', 'security')\n setDefaultIfNotExists('class_name', 'HTTP Activity')\n setDefaultIfNotExists('category_name', 'Network Activity')\n setDefaultIfNotExists('type_name', 'HTTP Activity: Other')\n setDefaultIfNotExists('type_uid', 400299)\n setDefaultIfNotExists('activity_id', 99)\n setDefaultIfNotExists('activity_name', event.eventName or '')\n setDefaultIfNotExists('event.type', event.eventName or '')\n setDefaultIfNotExists('severity_id', 99)\n setDefaultIfNotExists('status_id', 99)\n setDefaultIfNotExists('status', 'Other')\n \n -- Initialize observables array\n local observables = {}\n \n -- Add IP address observable\n if event.sourceIPAddress then\n table.insert(observables, {\n type_id = 2, \n type = 'IP Address', \n name = 'src_endpoint.ip', \n value = event.sourceIPAddress\n })\n end\n \n -- Add ARN observable\n local arn = getNestedField(event, 'userIdentity.arn')\n if arn then\n table.insert(observables, {\n type_id = 99,\n type = 'Other',\n name = 'unmapped.userIdentity.arn',\n value = arn\n })\n end\n \n setNestedField(result, 'observables', observables)\n \n -- Add unmapped fields\n local unmapped = {}\n \n -- Add specific unmapped fields that should be preserved\n if event.eventName then unmapped.eventName = event.eventName end\n if event.readOnly ~= nil then unmapped.readOnly = event.readOnly end\n if event.resources then unmapped.resources = event.resources end\n if event.eventType then unmapped.eventType = event.eventType end\n if event.managementEvent ~= nil then unmapped.managementEvent = event.managementEvent end\n if event.sharedEventID then unmapped.sharedEventID = event.sharedEventID end\n \n -- Add additionalEventData fields as unmapped\n if event.additionalEventData then\n if event.additionalEventData.SignatureVersion then unmapped[\"additionalEventData.SignatureVersion\"] = event.additionalEventData.SignatureVersion end\n if event.additionalEventData.CipherSuite then unmapped[\"additionalEventData.CipherSuite\"] = event.additionalEventData.CipherSuite end\n if event.additionalEventData.bytesTransferredIn then unmapped[\"additionalEventData.bytesTransferredIn\"] = event.additionalEventData.bytesTransferredIn end\n if event.additionalEventData.AuthenticationMethod then unmapped[\"additionalEventData.AuthenticationMethod\"] = event.additionalEventData.AuthenticationMethod end\n if event.additionalEventData.bytesTransferredOut then unmapped[\"additionalEventData.bytesTransferredOut\"] = event.additionalEventData.bytesTransferredOut end\n end\n \n -- Add all unmapped fields automatically\n -- Build comprehensive set of all mapped paths\n local mappedPaths = {}\n \n -- Add computed/mapped-by-logic paths\n mappedPaths['userIdentity.sessionContext.attributes.creationDate'] = true\n mappedPaths['responseElements.credentials.expiration'] = true\n \n -- Add all fieldMappings paths\n for _, mapping in ipairs(fieldMappings) do\n if mapping.type == \"direct\" then\n mappedPaths[mapping.source] = true\n elseif mapping.type == \"priority\" then\n if mapping.source1 then mappedPaths[mapping.source1] = true end\n if mapping.source2 then mappedPaths[mapping.source2] = true end\n if mapping.source3 then mappedPaths[mapping.source3] = true end\n end\n end\n \n for k, v in pairs(event) do\n -- Check if k is used as a source field in mapping\n local is_mapped = mappedPaths[k] == true\n \n -- Filter out Vector-specific fields that shouldn't be in unmapped\n local is_vector_field = (k == \"_ob\" or k == \"site_id\" or k == \"timestamp\")\n \n if not is_mapped and not is_vector_field then\n if type(v) == \"table\" then\n -- For nested objects, filter out only the mapped fields\n local function filterMappedFields(obj, prefix)\n local out = {}\n for nestedKey, nestedValue in pairs(obj) do\n local fullPath = prefix == \"\" and nestedKey or prefix .. \".\" .. nestedKey\n local isNestedMapped = mappedPaths[fullPath] == true\n \n -- Only include unmapped fields\n if not isNestedMapped then\n if type(nestedValue) == \"table\" then\n local child = filterMappedFields(nestedValue, fullPath)\n if next(child) then\n out[nestedKey] = child\n end\n else\n out[nestedKey] = nestedValue\n end\n end\n end\n return out\n end\n \n local filteredObj = filterMappedFields(v, k)\n if next(filteredObj) then\n unmapped[k] = filteredObj\n end\n else\n -- Only add non-null and non-empty values\n if v ~= nil and v ~= \"\" then\n unmapped[k] = v\n end\n end\n end\n end\n \n \n if next(unmapped) then\n setNestedField(result, 'unmapped', unmapped)\n end\n \n -- Create message field with original event\n local cleanEvent = {}\n for key, value in pairs(event) do\n if key ~= \"_ob\" and key ~= \"timestamp\" then\n cleanEvent[key] = value\n end\n end\n \n if event.responseElements == nil then\n cleanEvent.responseElements = \"NULL_PLACEHOLDER\"\n end\n \n -- Add missing fields that should be in the message\n if event.readOnly == nil then\n cleanEvent.readOnly = true -- Default for CloudTrail events\n end\n if event.eventType == nil then\n cleanEvent.eventType = \"AwsApiCall\" -- Default for CloudTrail events\n end\n if event.managementEvent == nil then\n cleanEvent.managementEvent = true -- Default for CloudTrail events\n end\n \n \n -- Flatten result\n local flattened = {}\n flattenObject(result, \"\", flattened)\n flattened.message = encodeJson(cleanEvent, \"root\")\n \n return flattened\nend\n\n-- Simplified helper functions\nfunction setNestedField(obj, path, value)\n if value == nil or path == nil or path == '' then return end\n local keys = {}\n for key in string.gmatch(path, '[^.]+') do\n if key and key ~= '' then\n table.insert(keys, key)\n end\n end\n if #keys == 0 then return end\n \n local current = obj\n for i = 1, #keys - 1 do\n local key = keys[i]\n if current[key] == nil then\n current[key] = {}\n end\n current = current[key]\n end\n current[keys[#keys]] = value\nend\n\nfunction getNestedField(obj, path)\n if obj == nil or path == nil or path == '' then return nil end\n local keys = {}\n for key in string.gmatch(path, '[^.]+') do\n if key and key ~= '' then\n table.insert(keys, key)\n end\n end\n if #keys == 0 then return nil end\n \n local current = obj\n for _, key in ipairs(keys) do\n if current == nil or current[key] == nil then\n return nil\n end\n current = current[key]\n end\n return current\nend\n\nfunction flattenObject(obj, prefix, result)\n prefix = prefix or \"\"\n if type(obj) ~= \"table\" then\n if prefix ~= \"\" then\n result[prefix] = obj\n end\n return\n end\n \n for key, value in pairs(obj) do\n local newKey = prefix == \"\" and key or prefix .. \".\" .. key\n if type(value) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(value) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n if isArray and maxIndex > 0 then\n local arrayValues = {}\n for i = 1, maxIndex do\n if value[i] ~= nil then\n table.insert(arrayValues, value[i])\n end\n end\n result[newKey] = arrayValues\n else\n flattenObject(value, newKey, result)\n end\n else\n result[newKey] = value\n end\n end\nend\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/metadata.yaml b/pipelines/community/transform_ocsf/aws_cloudtrail/metadata.yaml deleted file mode 100644 index f03dbf8..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/metadata.yaml +++ /dev/null @@ -1,52 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Aws Cloudtrail. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: aws - dataSource: Aws Cloudtrail - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "Other - {Explain: object store (S3) with SQS/SNS notifications}" - auth_type: "IAM Role" - sample_record: "{\n \"eventCategory\": \"Management\",\n \"eventName\": \"CreateUser\",\n \"eventSource\"\ - : \"iam.amazonaws.com\",\n \"eventTime\": \"2026-04-20T03:40:52Z\",\n \"eventVersion\": \"1.09\"\ - ,\n \"eventID\": \"4ad68099-cad0-4172-8711-dd15c4d352c9\",\n \"eventType\": \"AwsApiCall\",\n \"\ - awsRegion\": \"us-west-2\",\n \"readOnly\": false,\n \"managementEvent\": true,\n \"recipientAccountId\"\ - : \"987654321098\",\n \"sourceIPAddress\": \"65.40.62.221\",\n \"userAgent\": \"aws-cli/2.15.9 Python/3.11.4\ - \ Linux/5.10\",\n \"tlsDetails\": {\n \"tlsVersion\": \"TLSv1.2\",\n \"cipherSuite\": \"ECDHE-RSA-AES128-GCM-SHA256\"\ - ,\n \"clientProvidedHostHeader\": \"iam.amazonaws.com\"\n },\n \"userIdentity\": {\n \"type\"\ - : \"IAMUser\",\n \"principalId\": \"AIDAMEDICAL6904\",\n \"arn\": \"arn:aws:iam::987654321098:user/karen.martinez\"\ - ,\n \"accountId\": \"987654321098\",\n \"accessKeyId\": \"AKIA2E99153C98AA4B52\",\n \"userName\"\ - : \"karen.martinez\",\n \"sessionContext\": {\n \"sessionIssuer\": {\n \"type\": \"\ - Role\",\n \"principalId\": \"AROANURSE\",\n \"arn\": \"arn:aws:iam::987654321098:role/nurse\"\ - ,\n \"userName\": \"nurse\",\n \"accountId\": \"987654321098\"\n },\n \"attributes\"\ - : {\n \"creationDate\": \"2026-04-20T02:46:52Z\",\n \"mfaAuthenticated\": \"true\"\n\ - \ }\n }\n },\n \"requestID\": \"c32fabb4-8295-49e0-9cb0-8a1833ce49fa\",\n \"requestParameters\"\ - : {\n \"durationSeconds\": 900,\n \"roleArn\": \"arn:aws:iam::987654321098:role/nurse\",\n \ - \ \"roleSessionName\": \"medical-session\",\n \"externalId\": \"a3b57b57-e05c-46ff-9007-2402db4e6004\"\ - ,\n \"userN" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: aws, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/sample.json b/pipelines/community/transform_ocsf/aws_cloudtrail/sample.json deleted file mode 100644 index 4a46ab8..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/sample.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "eventCategory": "Management", - "eventName": "CreateUser", - "eventSource": "iam.amazonaws.com", - "eventTime": "2026-04-20T03:40:52Z", - "eventVersion": "1.09", - "eventID": "4ad68099-cad0-4172-8711-dd15c4d352c9", - "eventType": "AwsApiCall", - "awsRegion": "us-west-2", - "readOnly": false, - "managementEvent": true, - "recipientAccountId": "987654321098", - "sourceIPAddress": "65.40.62.221", - "userAgent": "aws-cli/2.15.9 Python/3.11.4 Linux/5.10", - "tlsDetails": { - "tlsVersion": "TLSv1.2", - "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256", - "clientProvidedHostHeader": "iam.amazonaws.com" - }, - "userIdentity": { - "type": "IAMUser", - "principalId": "AIDAMEDICAL6904", - "arn": "arn:aws:iam::987654321098:user/karen.martinez", - "accountId": "987654321098", - "accessKeyId": "AKIA2E99153C98AA4B52", - "userName": "karen.martinez", - "sessionContext": { - "sessionIssuer": { - "type": "Role", - "principalId": "AROANURSE", - "arn": "arn:aws:iam::987654321098:role/nurse", - "userName": "nurse", - "accountId": "987654321098" - }, - "attributes": { - "creationDate": "2026-04-20T02:46:52Z", - "mfaAuthenticated": "true" - } - } - }, - "requestID": "c32fabb4-8295-49e0-9cb0-8a1833ce49fa", - "requestParameters": { - "durationSeconds": 900, - "roleArn": "arn:aws:iam::987654321098:role/nurse", - "roleSessionName": "medical-session", - "externalId": "a3b57b57-e05c-46ff-9007-2402db4e6004", - "userName": "contractor.johnson", - "tags": [ - { - "Key": "Office", - "Value": "Chicago" - }, - { - "Key": "Department", - "Value": "Science" - } - ] - }, - "responseElements": { - "assumedRoleUser": { - "assumedRoleId": "AROANURSE:medical-session", - "arn": "arn:aws:sts::987654321098:assumed-role/nurse/medical-session" - }, - "credentials": { - "accessKeyId": "ASIADD7D979CACA44C1E", - "sessionToken": "IQoJb3JpZ2luX2VjEJ7//////////wEaCXVzLWVhc3QtMSJHMEUCIQD0b62c804fb42418e83d6fe294790a301", - "expiration": "2026-04-20T04:40:52Z" - }, - "sourceIdentity": "karen.martinez" - }, - "sharedEventID": "3d06df36-b385-486b-969f-f8f427a3f731", - "vpcEndpointId": "vpce-medical-55b991bf5", - "resources": [ - { - "accountId": "987654321098", - "type": "AWS::S3::Bucket", - "ARN": "arn:aws:s3:::backup-sensor-data" - } - ], - "additionalEventData": { - "SignatureVersion": "SigV4", - "CipherSuite": "ECDHE-RSA-AES256-GCM-SHA384", - "bytesTransferredIn": 0, - "bytesTransferredOut": 6607, - "AuthenticationMethod": "AuthHeader", - "x-amz-id-2": "6cd81f33b33044aa9040972beaf35e0e" - }, - "message": "karen.martinez from medical executed CreateUser on iam.amazonaws.com" -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/serializer.lua b/pipelines/community/transform_ocsf/aws_cloudtrail/serializer.lua deleted file mode 100644 index 47611c6..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/serializer.lua +++ /dev/null @@ -1,552 +0,0 @@ - -local FEATURES = { - IGNORE_UNKNOWN_EVENT = true, -} - --- Field ordering templates for consistent JSON serialization -local FIELD_ORDERS = { - root = {"eventVersion", "userIdentity", "eventTime", "eventSource", "eventName", "awsRegion", "sourceIPAddress", "userAgent", "requestParameters", "responseElements", "additionalEventData", "requestID", "eventID", "readOnly", "resources", "eventType", "managementEvent", "recipientAccountId", "sharedEventID", "eventCategory", "tlsDetails", "errorCode", "errorMessage", "vpcEndpointId", "apiVersion", "message", "time", "insightDetails", "class_uid", "category_uid", "metadata", "dataSource"}, - userIdentity = {"accountId", "principalId", "accessKeyId", "userName", "type", "invokedBy", "sessionContext"}, - sessionContext = {"sessionIssuer", "attributes"}, - sessionIssuer = {"type", "principalId", "arn", "accountId", "userName"}, - attributes = {"creationDate"}, - requestParameters = {"durationSeconds", "externalId", "bucketName", "Host", "instanceId", "availabilityZone", "requestContext"}, - requestContext = {"awsAccountId"}, - responseElements = {"credentials"}, - credentials = {"accessKeyId", "expiration"}, - additionalEventData = {"SignatureVersion", "CipherSuite", "bytesTransferredIn", "AuthenticationMethod", "x-amz-id-2", "bytesTransferredOut"}, - resources = {"accountId", "type", "ARN"}, - tlsDetails = {"tlsVersion", "cipherSuite"}, - insightDetails = {"eventSource", "insightContext"}, - insightContext = {"statistics"}, - statistics = {"insightDuration"}, - metadata = {"product"}, - product = {"name", "vendor_name"}, - dataSource = {"category", "vendor", "name"} -} - --- Convert ISO 8601 timestamp to Unix epoch milliseconds -function convertToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - - -- Parse ISO 8601 format: "2025-09-29T09:15:40Z" or "2025-09-29T09:15:40.123Z" - local year, month, day, hour, min, sec, ms = string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z") - - if year and month and day and hour and min and sec then - local t = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - - -- Get local time interpretation - local local_seconds = os.time(t) - - -- Get what this represents in UTC - local utc_date = os.date("!*t", local_seconds) - - local unix_seconds - -- Check if the UTC interpretation matches our input - if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and - utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and - utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then - -- We are already in UTC, use as-is - unix_seconds = local_seconds - else - -- Calculate the correct UTC timestamp - local utc_seconds = os.time(utc_date) - local offset = local_seconds - utc_seconds - unix_seconds = local_seconds + offset -- Add offset to get UTC - end - - -- Add milliseconds if present - local milli = 0 - if ms and ms ~= "" then - milli = tonumber((ms .. "000"):sub(1, 3)) -- pad/truncate to 3 digits - end - - return unix_seconds * 1000 + milli - end - - return nil -end - --- Optimized JSON encoding function with predefined ordering -function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray and maxIndex > 0 then - local items = {} - for i = 1, maxIndex do - -- Use the parent key for predefined ordering if available - local elementKey = key or tostring(i) - table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or "null") - end - return "[" .. table.concat(items, ", ") .. "]" - else - local items = {} - local fieldOrder = FIELD_ORDERS[key] or {} - - -- Phase 1: Process fields in predefined order - for _, fieldName in ipairs(fieldOrder) do - local v = obj[fieldName] - if v ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '": ' .. encodeJson(v, fieldName)) - end - end - - -- Phase 2: Process remaining fields - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. encodeJson(v, keyStr)) - end - end - - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -function processEvent(event) - - -- Check if eventType is missing and IGNORE_UNKNOWN_EVENT is enabled - if FEATURES.IGNORE_UNKNOWN_EVENT and (not event.eventType or event.eventType == "") then - return nil - end - - local result = {} - - -- Handle responseElements null case - if event.responseElements == "" then - event.responseElements = nil - end - - -- Direct field mappings for better performance - local function setField(sourcePath, targetPath) - local value = getNestedField(event, sourcePath) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - end - end - - -- Priority-based field mapping with fallback - local function setFieldWithPriority(priority1, priority2, priority3, targetPath) - local value = getNestedField(event, priority1) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - elseif priority2 then - value = getNestedField(event, priority2) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - elseif priority3 and priority3 ~= "" then - value = getNestedField(event, priority3) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - end - end - end - end - - - -- Field mapping table for better readability and maintainability - local fieldMappings = { - -- Basic CloudTrail fields - {type = "direct", source = "awsRegion", target = "cloud.region"}, - {type = "direct", source = "eventCategory", target = "metadata.product.feature.name"}, - {type = "direct", source = "eventID", target = "metadata.uid"}, - {type = "direct", source = "eventTime", target = "metadata.original_time"}, - {type = "direct", source = "eventVersion", target = "metadata.product.version"}, - {type = "direct", source = "recipientAccountId", target = "cloud.account.uid"}, - {type = "priority", source1 = "requestID", source2 = "requestParameters.externalId", source3 = "requestParameters.requestContext.awsAccountId", target = "api.request.uid"}, - {type = "direct", source = "sourceIPAddress", target = "src_endpoint.ip"}, - {type = "direct", source = "userAgent", target = "http_request.user_agent"}, - - -- User Identity fields - {type = "direct", source = "userIdentity.principalId", target = "actor.user.uid"}, - {type = "direct", source = "userIdentity.accessKeyId", target = "actor.user.credential_uid"}, - {type = "direct", source = "userIdentity.type", target = "actor.user.type"}, - {type = "direct", source = "userIdentity.invokedBy", target = "actor.invoked_by"}, - {type = "direct", source = "userIdentity.sessionContext.sessionIssuer.principalId", target = "actor.session.uid"}, - {type = "direct", source = "userIdentity.sessionContext.sessionIssuer.userName", target = "actor.session.issuer"}, - {type = "priority", source1 = "userIdentity.userName", source2 = "userIdentity.sessionContext.sessionIssuer.type", target = "actor.user.name"}, - {type = "priority", source1 = "userIdentity.accountId", source2 = "userIdentity.sessionContext.sessionIssuer.accountId", target = "actor.user.account.uid"}, - - -- API Information - {type = "direct", source = "errorCode", target = "api.response.error"}, - {type = "direct", source = "errorMessage", target = "api.response.error_message"}, - - -- Request Parameters - {type = "priority", source1 = "requestParameters.durationSeconds", source2 = "insightDetails.insightContext.statistics.insightDuration", target = "duration"}, - {type = "direct", source = "requestParameters.bucketName", target = "resources.name"}, - {type = "direct", source = "requestParameters.Host", target = "src_endpoint.hostname"}, - {type = "direct", source = "requestParameters.instanceId", target = "src_endpoint.instance_uid"}, - {type = "direct", source = "requestParameters.availabilityZone", target = "cloud.zone"}, - - -- Response Elements - {type = "direct", source = "responseElements.credentials.accessKeyId", target = "actor.session.credential_uid"}, - {type = "direct", source = "responseElements.credentials.expiration", target = "actor.session.expiration_time"}, - - -- Additional Event Data - {type = "direct", source = "additionalEventData.x-amz-id-2", target = "resources.uid"}, - {type = "direct", source = "tlsDetails.cipherSuite", target = "tls.cipher"}, - {type = "direct", source = "tlsDetails.tlsVersion", target = "tls.version"}, - - -- Additional fields - {type = "direct", source = "vpcEndpointId", target = "src_endpoint.uid"}, - {type = "direct", source = "apiVersion", target = "api.version"}, - {type = "direct", source = "message", target = "message"}, - {type = "priority", source1 = "eventSource", source2 = "insightDetails.eventSource", target = "api.service.name"}, - - -- Individual resource field mappings (for single resource events) - {type = "direct", source = "resources.accountId", target = "resource.account.uid"}, - {type = "direct", source = "resources.type", target = "resource.type"}, - {type = "direct", source = "resources.ARN", target = "resource.uid"}, - - -- OCSF field mappings (direct from input) - {type = "direct", source = "class_uid", target = "class_uid"}, - {type = "direct", source = "category_uid", target = "category_uid"} - } - - -- Process all field mappings in one iteration - for _, mapping in ipairs(fieldMappings) do - if mapping.type == "direct" then - setField(mapping.source, mapping.target) - elseif mapping.type == "priority" then - setFieldWithPriority(mapping.source1, mapping.source2, mapping.source3, mapping.target) - end - end - - -- Convert timestamps to milliseconds - local creationDate = getNestedField(event, 'userIdentity.sessionContext.attributes.creationDate') - if creationDate then - local convertedTime = convertToMilliseconds(creationDate) - if convertedTime then - setNestedField(result, 'actor.session.created_time', convertedTime) - end - end - - local expirationDate = getNestedField(event, 'responseElements.credentials.expiration') - if expirationDate then - local convertedTime = convertToMilliseconds(expirationDate) - if convertedTime then - setNestedField(result, 'actor.session.expiration_time', convertedTime) - end - end - - -- Convert eventTime to time field - local eventTime = getNestedField(event, 'eventTime') - if eventTime then - local convertedTime = convertToMilliseconds(eventTime) - if convertedTime then - -- we need to use milliseconds here to be compatible with S1 - setNestedField(result, 'time', convertedTime) - end - end - - -- Resources array handling - if event.resources and type(event.resources) == "table" then - local accountIds = {} - local types = {} - local arns = {} - - for _, resource in ipairs(event.resources) do - if resource.accountId then table.insert(accountIds, resource.accountId) end - if resource.type then table.insert(types, resource.type) end - if resource.ARN then table.insert(arns, resource.ARN) end - end - - if #accountIds > 0 then setNestedField(result, 'resource.account.uid', accountIds) end - if #types > 0 then setNestedField(result, 'resource.type', types) end - if #arns > 0 then setNestedField(result, 'resource.uid', arns) end - end - - -- Priority-based default OCSF fields (only set if not already present) - local function setDefaultIfNotExists(targetPath, defaultValue) - local existingValue = getNestedField(result, targetPath) - if existingValue == nil then - setNestedField(result, targetPath, defaultValue) - end - end - - -- Static category mapping - setDefaultIfNotExists('category_uid', 4) - - -- Static class mapping - setDefaultIfNotExists('class_uid', 4002) - - -- Set defaults only if fields don't already exist - setDefaultIfNotExists('metadata.product.name', 'CloudTrail') - setDefaultIfNotExists('metadata.product.vendor_name', 'AWS') - setDefaultIfNotExists('metadata.version', '1.0.0-rc3') - setDefaultIfNotExists('dataSource.vendor', 'AWS') - setDefaultIfNotExists('dataSource.name', 'CloudTrail') - setDefaultIfNotExists('dataSource.category', 'security') - setDefaultIfNotExists('class_name', 'HTTP Activity') - setDefaultIfNotExists('category_name', 'Network Activity') - setDefaultIfNotExists('type_name', 'HTTP Activity: Other') - setDefaultIfNotExists('type_uid', 400299) - setDefaultIfNotExists('activity_id', 99) - setDefaultIfNotExists('activity_name', event.eventName or '') - setDefaultIfNotExists('event.type', event.eventName or '') - setDefaultIfNotExists('severity_id', 99) - setDefaultIfNotExists('status_id', 99) - setDefaultIfNotExists('status', 'Other') - - -- Initialize observables array - local observables = {} - - -- Add IP address observable - if event.sourceIPAddress then - table.insert(observables, { - type_id = 2, - type = 'IP Address', - name = 'src_endpoint.ip', - value = event.sourceIPAddress - }) - end - - -- Add ARN observable - local arn = getNestedField(event, 'userIdentity.arn') - if arn then - table.insert(observables, { - type_id = 99, - type = 'Other', - name = 'unmapped.userIdentity.arn', - value = arn - }) - end - - setNestedField(result, 'observables', observables) - - -- Add unmapped fields - local unmapped = {} - - -- Add specific unmapped fields that should be preserved - if event.eventName then unmapped.eventName = event.eventName end - if event.readOnly ~= nil then unmapped.readOnly = event.readOnly end - if event.resources then unmapped.resources = event.resources end - if event.eventType then unmapped.eventType = event.eventType end - if event.managementEvent ~= nil then unmapped.managementEvent = event.managementEvent end - if event.sharedEventID then unmapped.sharedEventID = event.sharedEventID end - - -- Add additionalEventData fields as unmapped - if event.additionalEventData then - if event.additionalEventData.SignatureVersion then unmapped["additionalEventData.SignatureVersion"] = event.additionalEventData.SignatureVersion end - if event.additionalEventData.CipherSuite then unmapped["additionalEventData.CipherSuite"] = event.additionalEventData.CipherSuite end - if event.additionalEventData.bytesTransferredIn then unmapped["additionalEventData.bytesTransferredIn"] = event.additionalEventData.bytesTransferredIn end - if event.additionalEventData.AuthenticationMethod then unmapped["additionalEventData.AuthenticationMethod"] = event.additionalEventData.AuthenticationMethod end - if event.additionalEventData.bytesTransferredOut then unmapped["additionalEventData.bytesTransferredOut"] = event.additionalEventData.bytesTransferredOut end - end - - -- Add all unmapped fields automatically - -- Build comprehensive set of all mapped paths - local mappedPaths = {} - - -- Add computed/mapped-by-logic paths - mappedPaths['userIdentity.sessionContext.attributes.creationDate'] = true - mappedPaths['responseElements.credentials.expiration'] = true - - -- Add all fieldMappings paths - for _, mapping in ipairs(fieldMappings) do - if mapping.type == "direct" then - mappedPaths[mapping.source] = true - elseif mapping.type == "priority" then - if mapping.source1 then mappedPaths[mapping.source1] = true end - if mapping.source2 then mappedPaths[mapping.source2] = true end - if mapping.source3 then mappedPaths[mapping.source3] = true end - end - end - - for k, v in pairs(event) do - -- Check if k is used as a source field in mapping - local is_mapped = mappedPaths[k] == true - - -- Filter out Vector-specific fields that shouldn't be in unmapped - local is_vector_field = (k == "_ob" or k == "site_id" or k == "timestamp") - - if not is_mapped and not is_vector_field then - if type(v) == "table" then - -- For nested objects, filter out only the mapped fields - local function filterMappedFields(obj, prefix) - local out = {} - for nestedKey, nestedValue in pairs(obj) do - local fullPath = prefix == "" and nestedKey or prefix .. "." .. nestedKey - local isNestedMapped = mappedPaths[fullPath] == true - - -- Only include unmapped fields - if not isNestedMapped then - if type(nestedValue) == "table" then - local child = filterMappedFields(nestedValue, fullPath) - if next(child) then - out[nestedKey] = child - end - else - out[nestedKey] = nestedValue - end - end - end - return out - end - - local filteredObj = filterMappedFields(v, k) - if next(filteredObj) then - unmapped[k] = filteredObj - end - else - -- Only add non-null and non-empty values - if v ~= nil and v ~= "" then - unmapped[k] = v - end - end - end - end - - - if next(unmapped) then - setNestedField(result, 'unmapped', unmapped) - end - - -- Create message field with original event - local cleanEvent = {} - for key, value in pairs(event) do - if key ~= "_ob" and key ~= "timestamp" then - cleanEvent[key] = value - end - end - - if event.responseElements == nil then - cleanEvent.responseElements = "NULL_PLACEHOLDER" - end - - -- Add missing fields that should be in the message - if event.readOnly == nil then - cleanEvent.readOnly = true -- Default for CloudTrail events - end - if event.eventType == nil then - cleanEvent.eventType = "AwsApiCall" -- Default for CloudTrail events - end - if event.managementEvent == nil then - cleanEvent.managementEvent = true -- Default for CloudTrail events - end - - - -- Flatten result - local flattened = {} - flattenObject(result, "", flattened) - flattened.message = encodeJson(cleanEvent, "root") - - return flattened -end - --- Simplified helper functions -function setNestedField(obj, path, value) - if value == nil or path == nil or path == '' then return end - local keys = {} - for key in string.gmatch(path, '[^.]+') do - if key and key ~= '' then - table.insert(keys, key) - end - end - if #keys == 0 then return end - - local current = obj - for i = 1, #keys - 1 do - local key = keys[i] - if current[key] == nil then - current[key] = {} - end - current = current[key] - end - current[keys[#keys]] = value -end - -function getNestedField(obj, path) - if obj == nil or path == nil or path == '' then return nil end - local keys = {} - for key in string.gmatch(path, '[^.]+') do - if key and key ~= '' then - table.insert(keys, key) - end - end - if #keys == 0 then return nil end - - local current = obj - for _, key in ipairs(keys) do - if current == nil or current[key] == nil then - return nil - end - current = current[key] - end - return current -end - -function flattenObject(obj, prefix, result) - prefix = prefix or "" - if type(obj) ~= "table" then - if prefix ~= "" then - result[prefix] = obj - end - return - end - - for key, value in pairs(obj) do - local newKey = prefix == "" and key or prefix .. "." .. key - if type(value) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(value) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray and maxIndex > 0 then - local arrayValues = {} - for i = 1, maxIndex do - if value[i] ~= nil then - table.insert(arrayValues, value[i]) - end - end - result[newKey] = arrayValues - else - flattenObject(value, newKey, result) - end - else - result[newKey] = value - end - end -end diff --git a/pipelines/community/transform_ocsf/aws_guardduty/aws_guardduty.json b/pipelines/community/transform_ocsf/aws_guardduty/aws_guardduty.json deleted file mode 100644 index b5701ba..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/aws_guardduty.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Aws Guardduty", - "grade": { - "letter": "F", - "score": 45, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Aws Guardduty. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "aws", - "source_name": "aws_guardduty", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "aws-guardduty-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- AWS GuardDuty OCSF 1.0.0 parser (ported from Python)\n\n-- Helper: split string by delimiter\nlocal function split(str, delimiter)\n local result = {}\n local escaped = delimiter:gsub(\"[%.%+%*%?%^%$%(%)%[%]%%]\", \"%%%1\")\n local pattern = \"([^\" .. escaped .. \"]+)\"\n for token in tostring(str):gmatch(pattern) do\n table.insert(result, token)\n end\n if #result == 0 and #tostring(str) > 0 then\n table.insert(result, str)\n end\n return result\nend\n\n-- Helper: safely access nested keys: keys is array, obj is table\nlocal function getByPath(obj, keys)\n local current = obj\n for _, k in ipairs(keys) do\n if current ~= nil and type(current) == \"table\" then\n current = current[k]\n else\n return nil\n end\n end\n return current\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n\t return value\n end\n local copy = {}\n for k, v in pairs(value) do\n\t if not (ignoreKeys and ignoreKeys[k]) then\n\t\t copy[k] = deepCopy(v, ignoreKeys)\n\t end\n end\n return copy\nend\n\n-- Common mapping applied before specific finders\nlocal function common_mapping(event, site_id)\n event[\"dataSource\"] = { category = \"security\", name = \"AWS GuardDuty\", vendor = \"AWS\" }\n event[\"metadata\"] = { version = \"1.0.0\", product = { name = \"AWS GuardDuty\", vendor_name = \"AWS\" } }\n event[\"cloudProvider\"] = \"AWS\"\n event[\"cloudAccountType\"] = \"AWS Account\"\n event[\"accountTypeId\"] = 10\n if site_id ~= nil and site_id ~= \"\" then\n event[\"site\"] = { id = site_id }\n end\n local sev = event[\"severity\"]\n if type(sev) == \"number\" then\n if sev >= 1.0 and sev <= 2.9 then\n event[\"severity_id\"] = 2\n elseif sev >= 4.0 and sev <= 6.9 then\n event[\"severity_id\"] = 3\n elseif sev >= 7.0 and sev <= 8.9 then\n event[\"severity_id\"] = 4\n else\n event[\"severity_id\"] = 0\n end\n end\nend\n\n-- After mapping, convert lat/lon into coordinates and remove lat/lon fields based on activity_name\nlocal function coordinates_mapping(parsed)\n local lat = parsed[\"lat\"]\n local lon = parsed[\"lon\"]\n if (type(lat) == \"number\" or type(lat) == \"string\") or (type(lon) == \"number\" or type(lon) == \"string\") then\n local latnum = tonumber(lat)\n local lonnum = tonumber(lon)\n if latnum and lonnum then\n local coords = { latnum, lonnum }\n local activity = parsed[\"activity_name\"]\n if activity == \"EC2 finding types\" or\n activity == \"EKS Runtime Monitoring finding types\" or\n activity == \"Kubernetes Audit Logs finding types\" or\n activity == \"RDS Protection finding types\" or\n activity == \"S3 finding types\" then\n parsed[\"src_endpoint.location.coordinates\"] = coords\n parsed[\"lat\"], parsed[\"lon\"] = nil, nil\n elseif activity == \"IAM finding types\" or activity == \"Lambda Protection finding types\" then\n parsed[\"device.location.coordinates\"] = coords\n parsed[\"lat\"], parsed[\"lon\"] = nil, nil\n end\n end\n end\nend\n\nlocal function flatten_table(tbl, prefix, out)\n out = out or {}\n prefix = prefix or \"\"\n local hasArrayElements = false\n \n for k, v in pairs(tbl) do\n if type(k) == \"number\" then\n hasArrayElements = true\n break\n end\n end\n \n if hasArrayElements then\n -- This is an array, store it as-is\n if prefix ~= \"\" then\n out[prefix] = tbl\n end\n return out\n end\n \n for k, v in pairs(tbl) do\n local key = prefix ~= \"\" and (prefix .. \".\" .. k) or k\n if type(v) == \"table\" then\n flatten_table(v, key, out)\n else\n out[key] = v\n end\n end\n \n return out\nend\n\n-- Apply a mapping table of { [source_dotted] = target_dotted }\nlocal function apply_mapping(event, mapping)\n local out = {}\n for src, dst in pairs(mapping) do\n local val = getByPath(event, split(src, \".\"))\n if val ~= nil then\n out[dst] = val\n end\n end\n\n -- NEW: auto-capture unmapped fields\n local copy = deepCopy(event)\n local flattenCopy = flatten_table(copy)\n for key, val in pairs(flattenCopy) do\n if mapping[key] == nil and val ~= nil and val ~= \"\" and val ~= \"[]\" and val ~= \"()\" and val ~= \"{}\" then\n out[\"unmapped.\" .. key] = val\n end\n end\n return out\nend\n\nlocal EC2_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\",\"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"instanceDetails\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\", \n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\",\n \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"networkConnectionAction\"\n },\n networkConnectionAction = {\n \"connectionDirection\", \"localIpDetails\", \"remoteIpDetails\", \"remotePortDetails\", \n \"localPortDetails\", \"protocol\", \"blocked\"\n },\n localIpDetails = {\n \"ipAddressV4\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n remotePortDetails = {\n \"port\", \"portName\"\n },\n localPortDetails = {\n \"port\", \"portName\"\n },\n additionalInfo = {\n \"threatListName\", \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal EKS_RUNTIME_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"eksClusterDetails\", \"kubernetesDetails\", \"containerDetails\", \"instanceDetails\"\n },\n eksClusterDetails = {\n \"name\", \"arn\", \"createdAt\", \"vpcId\", \"status\", \"tags\"\n },\n tags = {\n \"key\", \"value\"\n },\n kubernetesDetails = {\n \"kubernetesWorkloadDetails\"\n },\n kubernetesWorkloadDetails = {\n \"name\", \"namespace\", \"type\", \"uid\"\n },\n containerDetails = {\n \"id\", \"name\", \"image\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"runtimeDetails\", \"featureName\", \"resourceRole\",\n \"additionalInfo\", \"evidence\", \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"dnsRequestAction\"\n },\n dnsRequestAction = {\n \"domain\", \"protocol\", \"blocked\", \"domainWithSuffix\"\n },\n runtimeDetails = {\n \"process\"\n },\n process = {\n \"pid\", \"name\", \"uuid\", \"executablePath\", \"executableSha256\", \"cmdLine\", \"user\", \"euid\",\n \"userId\", \"pwd\", \"startTime\", \"parentUuid\", \"lineage\"\n },\n lineage = {\n \"pid\", \"uuid\", \"executablePath\", \"euid\", \"parentUuid\"\n },\n additionalInfo = {\n \"threatListName\", \"sample\", \"agentDetails\", \"value\", \"type\"\n },\n agentDetails = {\n \"agentVersion\", \"agentId\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal IAM_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"accessKeyDetails\", \"instanceDetails\"\n },\n accessKeyDetails = {\n \"accessKeyId\", \"principalId\", \"userType\", \"userName\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"awsApiCallAction\"\n },\n awsApiCallAction = {\n \"api\", \"serviceName\", \"callerType\", \"errorCode\", \"remoteIpDetails\", \"affectedResources\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"userAgent\", \"anomalies\", \"profiledBehavior\", \"unusualBehavior\", \"sample\", \"value\", \"type\"\n },\n userAgent = {\n \"fullUserAgent\", \"userAgentCategory\"\n },\n anomalies = {\n \"anomalousAPIs\"\n },\n profiledBehavior = {\n \"rareProfiledAPIsAccountProfiling\", \"infrequentProfiledAPIsAccountProfiling\",\n \"frequentProfiledAPIsAccountProfiling\", \"rareProfiledAPIsUserIdentityProfiling\",\n \"infrequentProfiledAPIsUserIdentityProfiling\", \"frequentProfiledAPIsUserIdentityProfiling\",\n \"rareProfiledUserTypesAccountProfiling\", \"infrequentProfiledUserTypesAccountProfiling\",\n \"frequentProfiledUserTypesAccountProfiling\", \"rareProfiledUserNamesAccountProfiling\",\n \"infrequentProfiledUserNamesAccountProfiling\", \"frequentProfiledUserNamesAccountProfiling\",\n \"rareProfiledASNsAccountProfiling\", \"infrequentProfiledASNsAccountProfiling\",\n \"frequentProfiledASNsAccountProfiling\", \"rareProfiledASNsUserIdentityProfiling\",\n \"infrequentProfiledASNsUserIdentityProfiling\", \"frequentProfiledASNsUserIdentityProfiling\",\n \"rareProfiledUserAgentsAccountProfiling\", \"infrequentProfiledUserAgentsAccountProfiling\",\n \"frequentProfiledUserAgentsAccountProfiling\", \"rareProfiledUserAgentsUserIdentityProfiling\",\n \"infrequentProfiledUserAgentsUserIdentityProfiling\", \"frequentProfiledUserAgentsUserIdentityProfiling\"\n },\n unusualBehavior = {\n \"unusualAPIsAccountProfiling\", \"unusualAPIsUserIdentityProfiling\",\n \"unusualUserTypesAccountProfiling\", \"unusualUserNamesAccountProfiling\",\n \"unusualASNsAccountProfiling\", \"unusualASNsUserIdentityProfiling\",\n \"unusualUserAgentsAccountProfiling\", \"unusualUserAgentsUserIdentityProfiling\",\n \"isUnusualUserIdentity\"\n },\n}\n\nlocal KUBERNETES_AUDIT_LOGS_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"eksClusterDetails\", \"kubernetesDetails\", \"accessKeyDetails\"\n },\n eksClusterDetails = {\n \"name\", \"arn\", \"createdAt\", \"vpcId\", \"status\", \"tags\"\n },\n tags = {\n \"key\", \"value\"\n },\n kubernetesDetails = {\n \"kubernetesWorkloadDetails\", \"kubernetesUserDetails\"\n },\n kubernetesWorkloadDetails = {\n \"name\", \"namespace\", \"type\", \"uid\"\n },\n kubernetesUserDetails = {\n \"username\", \"uid\", \"groups\", \"sessionName\"\n },\n accessKeyDetails = {\n \"accessKeyId\", \"principalId\", \"userType\", \"userName\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"kubernetesApiCallAction\"\n },\n kubernetesApiCallAction = {\n \"requestUri\", \"verb\", \"sourceIPs\", \"userAgent\", \"remoteIpDetails\", \"statusCode\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"threatName\", \"threatListName\", \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal LAMBDA_PROTECTION_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"lambdaDetails\"\n },\n lambdaDetails = {\n \"functionArn\", \"functionName\", \"description\", \"lastModifiedAt\", \"revisionId\",\n \"functionVersion\", \"role\", \"vpcConfig\", \"tags\"\n },\n vpcConfig = {\n \"vpcId\", \"securityGroups\", \"subnetIds\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"networkConnectionAction\"\n },\n networkConnectionAction = {\n \"connectionDirection\", \"remoteIpDetails\", \"remotePortDetails\", \"protocol\", \"blocked\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n remotePortDetails = {\n \"port\", \"portName\"\n },\n additionalInfo = {\n \"unusualProtocol\", \"threatListName\", \"unusual\", \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal MALWARE_PROTECTION_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"instanceDetails\", \"ebsVolumeDetails\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n ebsVolumeDetails = {\n \"scannedVolumeDetails\", \"skippedVolumeDetails\"\n },\n scannedVolumeDetails = {\n \"volumeArn\", \"volumeType\", \"deviceName\", \"volumeSizeInGB\", \"encryptionType\", \"snapshotArn\", \"kmsKeyArn\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"featureName\", \"ebsVolumeScanDetails\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n ebsVolumeScanDetails = {\n \"scanId\", \"scanStartedAt\", \"scanCompletedAt\", \"scanType\", \"triggerFindingId\", \"sources\", \"scanDetections\"\n },\n scanDetections = {\n \"scannedItemCount\", \"threatsDetectedItemCount\", \"highestSeverityThreatDetails\", \"threatDetectedByName\"\n },\n scannedItemCount = {\n \"totalGb\", \"files\", \"volumes\"\n },\n threatsDetectedItemCount = {\n \"files\"\n },\n highestSeverityThreatDetails = {\n \"severity\", \"threatName\", \"count\"\n },\n threatDetectedByName = {\n \"itemCount\", \"uniqueThreatNameCount\", \"shortened\", \"threatNames\"\n },\n threatNames = {\n \"name\", \"severity\", \"itemCount\", \"filePaths\"\n },\n filePaths = {\n \"filePath\", \"fileName\", \"volumeArn\", \"hash\"\n },\n additionalInfo = {\n \"sample\", \"value\", \"type\"\n },\n}\n\nlocal RDS_PROTECTION_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"rdsDbInstanceDetails\", \"rdsDbUserDetails\"\n },\n rdsDbInstanceDetails = {\n \"dbInstanceIdentifier\", \"engine\", \"engineVersion\", \"dbClusterIdentifier\", \"dbInstanceArn\", \"tags\"\n },\n tags = {\n \"key\", \"value\"\n },\n rdsDbUserDetails = {\n \"user\", \"application\", \"database\", \"ssl\", \"authMethod\"\n },\n service = {\n \"action\", \"additionalInfo\", \"resourceRole\", \"evidence\", \"count\", \"detectorId\",\n \"eventFirstSeen\", \"eventLastSeen\", \"serviceName\", \"archived\"\n },\n action = {\n \"actionType\", \"rdsLoginAttemptAction\"\n },\n rdsLoginAttemptAction = {\n \"remoteIpDetails\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal S3_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"accessKeyDetails\", \"s3BucketDetails\", \"instanceDetails\"\n },\n accessKeyDetails = {\n \"accessKeyId\", \"principalId\", \"userType\", \"userName\"\n },\n s3BucketDetails = {\n \"arn\", \"name\", \"type\", \"createdAt\", \"owner\", \"tags\", \"defaultServerSideEncryption\", \"publicAccess\"\n },\n owner = {\n \"id\"\n },\n tags = {\n \"key\", \"value\"\n },\n defaultServerSideEncryption = {\n \"encryptionType\", \"kmsMasterKeyArn\"\n },\n publicAccess = {\n \"permissionConfiguration\", \"effectivePermission\"\n },\n permissionConfiguration = {\n \"bucketLevelPermissions\", \"accountLevelPermissions\"\n },\n bucketLevelPermissions = {\n \"accessControlList\", \"bucketPolicy\", \"blockPublicAccess\"\n },\n accessControlList = {\n \"allowsPublicReadAccess\", \"allowsPublicWriteAccess\"\n },\n bucketPolicy = {\n \"allowsPublicReadAccess\", \"allowsPublicWriteAccess\"\n },\n blockPublicAccess = {\n \"ignorePublicAcls\", \"restrictPublicBuckets\", \"blockPublicAcls\", \"blockPublicPolicy\"\n },\n accountLevelPermissions = {\n \"blockPublicAccess\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"awsApiCallAction\"\n },\n awsApiCallAction = {\n \"api\", \"serviceName\", \"callerType\", \"errorCode\", \"remoteIpDetails\", \"affectedResources\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"unusual\", \"sample\", \"value\", \"type\"\n },\n unusual = {\n \"hoursOfDay\", \"userNames\"\n },\n}\n\nlocal function get_msg_field_ordering(family)\n if family == \"ec2\" then return EC2_FIELD_ORDER end\n if family == \"runtime\" or family == \"eks_runtime\" or family == \"runtime_monitoring\" then return EKS_RUNTIME_FIELD_ORDER end\n if family == \"iam\" then return IAM_FIELD_ORDER end\n if family == \"kubernetes\" or family == \"k8s\" then return KUBERNETES_AUDIT_LOGS_FIELD_ORDER end\n if family == \"lambda\" then return LAMBDA_PROTECTION_FIELD_ORDER end\n if family == \"malware\" then return MALWARE_PROTECTION_FIELD_ORDER end\n if family == \"rds\" then return RDS_PROTECTION_FIELD_ORDER end\n if family == \"s3\" then return S3_FIELD_ORDER end\n return {}\nend\n\n-- OCSF Mappings (ported 1:1 from Python)\nlocal function ec2_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.instanceDetails.instanceId\"] = \"device.instance_uid\",\n [\"resource.instanceDetails.networkInterfaces\"] = \"device.network_interfaces\",\n [\"resource.instanceDetails.imageId\"] = \"device.image.uid\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.action.networkConnectionAction.localIpDetails.ipAddressV4\"] = \"src_endpoint.ip\",\n [\"service.action.networkConnectionAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.networkConnectionAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.networkConnectionAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.action.networkConnectionAction.localPortDetails.port\"] = \"src_endpoint.port\",\n [\"service.action.networkConnectionAction.protocol\"] = \"connection_info.protocol_name\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function runtime_monitoring_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.instanceDetails.instanceId\"] = \"device.instance_uid\",\n [\"resource.instanceDetails.networkInterfaces\"] = \"device.network_interfaces\",\n [\"resource.instanceDetails.imageId\"] = \"device.image.uid\",\n [\"service.runtimeDetails.process.pid\"] = \"actor.process.uid\",\n [\"service.runtimeDetails.process.name\"] = \"actor.process.name\",\n [\"service.runtimeDetails.process.user\"] = \"actor.user.name\",\n [\"service.runtimeDetails.process.userId\"] = \"actor.user.uid\",\n [\"service.runtimeDetails.process.parentUuid\"] = \"actor.process.parent_process.uid\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.featureName\"] = \"api.service.name\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function iam_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.accessKeyDetails.accessKeyId\"] = \"actor.session.credential_uid\",\n [\"resource.accessKeyDetails.principalId\"] = \"actor.session.uid\",\n [\"resource.accessKeyDetails.userName\"] = \"actor.user.name\",\n [\"resource.instanceDetails.instanceId\"] = \"device.instance_uid\",\n [\"resource.instanceDetails.networkInterfaces\"] = \"device.network_interfaces\",\n [\"resource.instanceDetails.imageId\"] = \"device.image.uid\",\n [\"service.action.awsApiCallAction.serviceName\"] = \"api.service.name\",\n [\"service.action.awsApiCallAction.remoteIpDetails.organization.isp\"] = \"device.location.isp\",\n [\"service.action.awsApiCallAction.remoteIpDetails.country.countryName\"] = \"device.location.country\",\n [\"service.action.awsApiCallAction.remoteIpDetails.city.cityName\"] = \"device.location.city\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function kubernetes_audit_log_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.kubernetesDetails.kubernetesUserDetails.username\"] = \"actor.user.name\",\n [\"resource.kubernetesDetails.kubernetesUserDetails.uid\"] = \"actor.user.uid\",\n [\"resource.kubernetesDetails.kubernetesUserDetails.groups\"] = \"actor.user.groups\",\n [\"resource.accessKeyDetails.accessKeyId\"] = \"actor.session.credential_uid\",\n [\"resource.accessKeyDetails.principalId\"] = \"actor.session.uid\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.action.kubernetesApiCallAction.requestUri\"] = \"http_request.url.url_string\",\n [\"service.action.kubernetesApiCallAction.verb\"] = \"http_request.http_method\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.action.kubernetesApiCallAction.statusCode\"] = \"status_code\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function lambda_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"service.action.networkConnectionAction.remoteIpDetails.organization.isp\"] = \"device.location.isp\",\n [\"service.action.networkConnectionAction.remoteIpDetails.country.countryName\"] = \"device.location.country\",\n [\"service.action.networkConnectionAction.remoteIpDetails.city.cityName\"] = \"device.location.city\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function malware_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"partition\"] = \"resources.cloud_partition\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"service.featureName\"] = \"api.service.name\",\n [\"service.ebsVolumeScanDetails.scanId\"] = \"api.service.uid\",\n [\"service.ebsVolumeScanDetails.scanStartedAt\"] = \"finding.first_seen_time\",\n [\"service.ebsVolumeScanDetails.scanCompletedAt\"] = \"finding.last_seen_time\",\n [\"service.ebsVolumeScanDetails.triggerFindingId\"] = \"finding.uid\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function rds_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function s3_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"partition\"] = \"resources.cloud_partition\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.accessKeyDetails.accessKeyId\"] = \"resources.user.credential_uid\",\n [\"resource.accessKeyDetails.userType\"] = \"resources.user.type\",\n [\"resource.accessKeyDetails.userName\"] = \"resources.user.name\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.action.awsApiCallAction.serviceName\"] = \"api.service.name\",\n [\"service.action.awsApiCallAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.awsApiCallAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.awsApiCallAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function unknown_findings_ocsf_mapping()\n return {\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\n-- Finding types constants\nlocal ec2_finding_types = {\n [\"Backdoor:EC2/C&CActivity.B\"] = true,\n [\"Backdoor:EC2/C&CActivity.B!DNS\"] = true,\n [\"Backdoor:EC2/DenialOfService.Dns\"] = true,\n [\"Backdoor:EC2/DenialOfService.Tcp\"] = true,\n [\"Backdoor:EC2/DenialOfService.Udp\"] = true,\n [\"Backdoor:EC2/DenialOfService.UdpOnTcpPorts\"] = true,\n [\"Backdoor:EC2/DenialOfService.UnusualProtocol\"] = true,\n [\"Backdoor:EC2/Spambot\"] = true,\n [\"Behavior:EC2/NetworkPortUnusual\"] = true,\n [\"Behavior:EC2/TrafficVolumeUnusual\"] = true,\n [\"CryptoCurrency:EC2/BitcoinTool.B\"] = true,\n [\"CryptoCurrency:EC2/BitcoinTool.B!DNS\"] = true,\n [\"DefenseEvasion:EC2/UnusualDNSResolver\"] = true,\n [\"DefenseEvasion:EC2/UnusualDoHActivity\"] = true,\n [\"DefenseEvasion:EC2/UnusualDoTActivity\"] = true,\n [\"Impact:EC2/AbusedDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/BitcoinDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/MaliciousDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/PortSweep\"] = true,\n [\"Impact:EC2/SuspiciousDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/WinRMBruteForce\"] = true,\n [\"Recon:EC2/PortProbeEMRUnprotectedPort\"] = true,\n [\"Recon:EC2/PortProbeUnprotectedPort\"] = true,\n [\"Recon:EC2/Portscan\"] = true,\n [\"Trojan:EC2/BlackholeTraffic\"] = true,\n [\"Trojan:EC2/BlackholeTraffic!DNS\"] = true,\n [\"Trojan:EC2/DGADomainRequest.B\"] = true,\n [\"Trojan:EC2/DGADomainRequest.C!DNS\"] = true,\n [\"Trojan:EC2/DNSDataExfiltration\"] = true,\n [\"Trojan:EC2/DriveBySourceTraffic!DNS\"] = true,\n [\"Trojan:EC2/DropPoint\"] = true,\n [\"Trojan:EC2/DropPoint!DNS\"] = true,\n [\"Trojan:EC2/PhishingDomainRequest!DNS\"] = true,\n [\"UnauthorizedAccess:EC2/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:EC2/MetadataDNSRebind\"] = true,\n [\"UnauthorizedAccess:EC2/RDPBruteForce\"] = true,\n [\"UnauthorizedAccess:EC2/SSHBruteForce\"] = true,\n [\"UnauthorizedAccess:EC2/TorClient\"] = true,\n [\"UnauthorizedAccess:EC2/TorRelay\"] = true\n}\n\nlocal runtime_monitoring_finding_types = {\n [\"CryptoCurrency:Runtime/BitcoinTool.B\"] = true,\n [\"Backdoor:Runtime/C&CActivity.B\"] = true,\n [\"UnauthorizedAccess:Runtime/TorRelay\"] = true,\n [\"UnauthorizedAccess:Runtime/TorClient\"] = true,\n [\"Trojan:Runtime/BlackholeTraffic\"] = true,\n [\"Trojan:Runtime/DropPoint\"] = true,\n [\"CryptoCurrency:Runtime/BitcoinTool.B!DNS\"] = true,\n [\"Backdoor:Runtime/C&CActivity.B!DNS\"] = true,\n [\"Trojan:Runtime/BlackholeTraffic!DNS\"] = true,\n [\"Trojan:Runtime/DropPoint!DNS\"] = true,\n [\"Trojan:Runtime/DGADomainRequest.C!DNS\"] = true,\n [\"Trojan:Runtime/DriveBySourceTraffic!DNS\"] = true,\n [\"Trojan:Runtime/PhishingDomainRequest!DNS\"] = true,\n [\"Impact:Runtime/AbusedDomainRequest.Reputation\"] = true,\n [\"Impact:Runtime/BitcoinDomainRequest.Reputation\"] = true,\n [\"Impact:Runtime/MaliciousDomainRequest.Reputation\"] = true,\n [\"Impact:Runtime/SuspiciousDomainRequest.Reputation\"] = true,\n [\"UnauthorizedAccess:Runtime/MetadataDNSRebind\"] = true,\n [\"Execution:Runtime/NewBinaryExecuted\"] = true,\n [\"PrivilegeEscalation:Runtime/DockerSocketAccessed\"] = true,\n [\"PrivilegeEscalation:Runtime/RuncContainerEscape\"] = true,\n [\"PrivilegeEscalation:Runtime/CGroupsReleaseAgentModified\"] = true,\n [\"DefenseEvasion:Runtime/ProcessInjection.Proc\"] = true,\n [\"DefenseEvasion:Runtime/ProcessInjection.Ptrace\"] = true,\n [\"DefenseEvasion:Runtime/ProcessInjection.VirtualMemoryWrite\"] = true,\n [\"Execution:Runtime/ReverseShell\"] = true,\n [\"DefenseEvasion:Runtime/FilelessExecution\"] = true,\n [\"Impact:Runtime/CryptoMinerExecuted\"] = true,\n [\"Execution:Runtime/NewLibraryLoaded\"] = true,\n [\"PrivilegeEscalation:Runtime/ContainerMountsHostDirectory\"] = true,\n [\"PrivilegeEscalation:Runtime/UserfaultfdUsage\"] = true\n}\n\nlocal iam_finding_types = {\n [\"CredentialAccess:IAMUser/AnomalousBehavior\"] = true,\n [\"DefenseEvasion:IAMUser/AnomalousBehavior\"] = true,\n [\"Discovery:IAMUser/AnomalousBehavior\"] = true,\n [\"Exfiltration:IAMUser/AnomalousBehavior\"] = true,\n [\"Impact:IAMUser/AnomalousBehavior\"] = true,\n [\"InitialAccess:IAMUser/AnomalousBehavior\"] = true,\n [\"PenTest:IAMUser/KaliLinux\"] = true,\n [\"PenTest:IAMUser/ParrotLinux\"] = true,\n [\"PenTest:IAMUser/PentooLinux\"] = true,\n [\"Persistence:IAMUser/AnomalousBehavior\"] = true,\n [\"Policy:IAMUser/RootCredentialUsage\"] = true,\n [\"PrivilegeEscalation:IAMUser/AnomalousBehavior\"] = true,\n [\"Recon:IAMUser/MaliciousIPCaller\"] = true,\n [\"Recon:IAMUser/MaliciousIPCaller.Custom\"] = true,\n [\"Recon:IAMUser/TorIPCaller\"] = true,\n [\"Stealth:IAMUser/CloudTrailLoggingDisabled\"] = true,\n [\"Stealth:IAMUser/PasswordPolicyChange\"] = true,\n [\"UnauthorizedAccess:IAMUser/ConsoleLoginSuccess.B\"] = true,\n [\"UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.InsideAWS\"] = true,\n [\"UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.OutsideAWS\"] = true,\n [\"UnauthorizedAccess:IAMUser/MaliciousIPCaller\"] = true,\n [\"UnauthorizedAccess:IAMUser/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:IAMUser/TorIPCaller\"] = true\n}\n\nlocal kubernetes_audit_logs_finding_types = {\n [\"CredentialAccess:Kubernetes/MaliciousIPCaller\"] = true,\n [\"CredentialAccess:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"CredentialAccess:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"CredentialAccess:Kubernetes/TorIPCaller\"] = true,\n [\"DefenseEvasion:Kubernetes/MaliciousIPCaller\"] = true,\n [\"DefenseEvasion:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"DefenseEvasion:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"DefenseEvasion:Kubernetes/TorIPCaller\"] = true,\n [\"Discovery:Kubernetes/MaliciousIPCaller\"] = true,\n [\"Discovery:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"Discovery:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"Discovery:Kubernetes/TorIPCaller\"] = true,\n [\"Execution:Kubernetes/ExecInKubeSystemPod\"] = true,\n [\"Impact:Kubernetes/MaliciousIPCaller\"] = true,\n [\"Impact:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"Impact:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"Impact:Kubernetes/TorIPCaller\"] = true,\n [\"Persistence:Kubernetes/ContainerWithSensitiveMount\"] = true,\n [\"Persistence:Kubernetes/MaliciousIPCaller\"] = true,\n [\"Persistence:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"Persistence:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"Persistence:Kubernetes/TorIPCaller\"] = true,\n [\"Policy:Kubernetes/AdminAccessToDefaultServiceAccount\"] = true,\n [\"Policy:Kubernetes/AnonymousAccessGranted\"] = true,\n [\"Policy:Kubernetes/ExposedDashboard\"] = true,\n [\"Policy:Kubernetes/KubeflowDashboardExposed\"] = true,\n [\"PrivilegeEscalation:Kubernetes/PrivilegedContainer\"] = true,\n [\"CredentialAccess:Kubernetes/AnomalousBehavior.SecretsAccessed\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleBindingCreated\"] = true,\n [\"Execution:Kubernetes/AnomalousBehavior.ExecInPod\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!PrivilegedContainer\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!ContainerWithSensitiveMount\"] = true,\n [\"Execution:Kubernetes/AnomalousBehavior.WorkloadDeployed\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleCreated\"] = true,\n [\"Discovery:Kubernetes/AnomalousBehavior.PermissionChecked\"] = true\n}\n\nlocal lambda_protection_finding_types = {\n [\"Backdoor:Lambda/C&CActivity.B\"] = true,\n [\"CryptoCurrency:Lambda/BitcoinTool.B\"] = true,\n [\"Trojan:Lambda/BlackholeTraffic\"] = true,\n [\"Trojan:Lambda/DropPoint\"] = true,\n [\"UnauthorizedAccess:Lambda/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:Lambda/TorClient\"] = true,\n [\"UnauthorizedAccess:Lambda/TorRelay\"] = true\n}\n\nlocal malware_protection_finding_types = {\n [\"Execution:EC2/MaliciousFile\"] = true,\n [\"Execution:ECS/MaliciousFile\"] = true,\n [\"Execution:Kubernetes/MaliciousFile\"] = true,\n [\"Execution:Container/MaliciousFile\"] = true,\n [\"Execution:EC2/SuspiciousFile\"] = true,\n [\"Execution:ECS/SuspiciousFile\"] = true,\n [\"Execution:Kubernetes/SuspiciousFile\"] = true,\n [\"Execution:Container/SuspiciousFile\"] = true\n}\n\nlocal rds_protection_finding_types = {\n [\"CredentialAccess:RDS/AnomalousBehavior.SuccessfulLogin\"] = true,\n [\"CredentialAccess:RDS/AnomalousBehavior.FailedLogin\"] = true,\n [\"CredentialAccess:RDS/AnomalousBehavior.SuccessfulBruteForce\"] = true,\n [\"CredentialAccess:RDS/MaliciousIPCaller.SuccessfulLogin\"] = true,\n [\"CredentialAccess:RDS/MaliciousIPCaller.FailedLogin\"] = true,\n [\"Discovery:RDS/MaliciousIPCaller\"] = true,\n [\"CredentialAccess:RDS/TorIPCaller.SuccessfulLogin\"] = true,\n [\"CredentialAccess:RDS/TorIPCaller.FailedLogin\"] = true,\n [\"Discovery:RDS/TorIPCaller\"] = true\n}\n\nlocal s3_finding_types = {\n [\"Discovery:S3/AnomalousBehavior\"] = true,\n [\"Discovery:S3/MaliciousIPCaller\"] = true,\n [\"Discovery:S3/MaliciousIPCaller.Custom\"] = true,\n [\"Discovery:S3/TorIPCaller\"] = true,\n [\"Exfiltration:S3/AnomalousBehavior\"] = true,\n [\"Exfiltration:S3/MaliciousIPCaller\"] = true,\n [\"Impact:S3/AnomalousBehavior.Delete\"] = true,\n [\"Impact:S3/AnomalousBehavior.Permission\"] = true,\n [\"Impact:S3/AnomalousBehavior.Write\"] = true,\n [\"Impact:S3/MaliciousIPCaller\"] = true,\n [\"PenTest:S3/KaliLinux\"] = true,\n [\"PenTest:S3/ParrotLinux\"] = true,\n [\"PenTest:S3/PentooLinux\"] = true,\n [\"Policy:S3/AccountBlockPublicAccessDisabled\"] = true,\n [\"Policy:S3/BucketAnonymousAccessGranted\"] = true,\n [\"Policy:S3/BucketBlockPublicAccessDisabled\"] = true,\n [\"Policy:S3/BucketPublicAccessGranted\"] = true,\n [\"Stealth:S3/ServerAccessLoggingDisabled\"] = true,\n [\"UnauthorizedAccess:S3/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:S3/TorIPCaller\"] = true\n}\n\n\n-- Per-category helpers (mirroring Python OCSFMapperHelper)\nlocal function ec2_findings(event, site_id)\n event[\"class_uid\"] = 4001\n event[\"class_name\"] = \"Network Activity\"\n event[\"category_uid\"] = 4\n event[\"category_name\"] = \"Network Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"EC2 finding types\"\n event[\"type_uid\"] = 400199\n event[\"type_name\"] = \"EC2 finding types\"\n event[\"eventId\"] = \"EC2 finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, ec2_findings_ocsf_mapping())\n -- enrich with fields that were set directly on event\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function runtime_monitoring_findings(event, site_id)\n event[\"class_uid\"] = 6003\n event[\"class_name\"] = \"API Activity\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"EKS Runtime Monitoring finding types\"\n event[\"type_uid\"] = 600399\n event[\"type_name\"] = \"EKS Runtime Monitoring finding types\"\n event[\"eventId\"] = \"EKS Runtime Monitoring finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, runtime_monitoring_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function iam_findings(event, site_id)\n event[\"class_uid\"] = 3004\n event[\"class_name\"] = \"Entity Management\"\n event[\"category_uid\"] = 3\n event[\"category_name\"] = \"Identity & Access Management\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"IAM finding types\"\n event[\"type_uid\"] = 300499\n event[\"type_name\"] = \"IAM finding types\"\n event[\"eventId\"] = \"IAM finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, iam_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function kubernetes_audit_log_findings(event, site_id)\n event[\"class_uid\"] = 6003\n event[\"class_name\"] = \"API Activity\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"Kubernetes Audit Logs finding types\"\n event[\"type_uid\"] = 600399\n event[\"type_name\"] = \"Kubernetes Audit Logs finding types\"\n event[\"eventId\"] = \"Kubernetes Audit Logs finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, kubernetes_audit_log_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function lambda_findings(event, site_id)\n event[\"class_uid\"] = 6002\n event[\"class_name\"] = \"Application Lifecycle\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"Lambda Protection finding types\"\n event[\"type_uid\"] = 600299\n event[\"type_name\"] = \"Lambda Protection finding types\"\n event[\"eventId\"] = \"Lambda Protection finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, lambda_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function malware_findings(event, site_id)\n event[\"class_uid\"] = 2001\n event[\"class_name\"] = \"Security Finding\"\n event[\"category_uid\"] = 2\n event[\"category_name\"] = \"Findings\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"Malware Protection finding types\"\n event[\"type_uid\"] = 200199\n event[\"type_name\"] = \"Malware Protection finding types\"\n event[\"eventId\"] = \"Malware Protection finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, malware_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n return flat\nend\n\nlocal function rds_findings(event, site_id)\n event[\"class_uid\"] = 3002\n event[\"class_name\"] = \"Authentication\"\n event[\"category_uid\"] = 3\n event[\"category_name\"] = \"Identity & Access Management\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"RDS Protection finding types\"\n event[\"type_uid\"] = 300299\n event[\"type_name\"] = \"RDS Protection finding types\"\n event[\"eventId\"] = \"RDS Protection finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, rds_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function s3_findings(event, site_id)\n event[\"class_uid\"] = 6003\n event[\"class_name\"] = \"API Activity\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"S3 finding types\"\n event[\"type_uid\"] = 600399\n event[\"type_name\"] = \"S3 finding types\"\n event[\"eventId\"] = \"S3 finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, s3_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function unknown_findings(event, site_id)\n event[\"class_uid\"] = 0\n event[\"class_name\"] = \"Base Event\"\n event[\"category_uid\"] = 0\n event[\"category_name\"] = \"Uncategorized\"\n event[\"activity_id\"] = 0\n event[\"activity_name\"] = \"Unknown\"\n event[\"type_uid\"] = 0\n event[\"type_name\"] = \"Unknown\"\n event[\"eventId\"] = \"Unknown\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, unknown_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n return flat\nend\n\n-- Dispatcher. family one of: \"ec2\",\"runtime\",\"iam\",\"kubernetes\",\"lambda\",\"malware\",\"rds\",\"s3\",\"unknown\"\nlocal function processGuardDutyEvent(event, site_id, family)\n local fam = tostring(family or \"unknown\"):lower()\n if fam == \"ec2\" then return ec2_findings(deepCopy(event), site_id) end\n if fam == \"runtime\" or fam == \"eks_runtime\" or fam == \"runtime_monitoring\" then return runtime_monitoring_findings(deepCopy(event), site_id) end\n if fam == \"iam\" then return iam_findings(deepCopy(event), site_id) end\n if fam == \"kubernetes\" or fam == \"k8s\" then return kubernetes_audit_log_findings(deepCopy(event), site_id) end\n if fam == \"lambda\" then return lambda_findings(deepCopy(event), site_id) end\n if fam == \"malware\" then return malware_findings(deepCopy(event), site_id) end\n if fam == \"rds\" then return rds_findings(deepCopy(event), site_id) end\n if fam == \"s3\" then return s3_findings(deepCopy(event), site_id) end\n return unknown_findings(deepCopy(event), site_id)\nend\n\n-- Best-effort automatic family detection from GuardDuty event structure\nlocal function detect_family(event)\n local type = getByPath(event, {\"type\"})\n if ec2_finding_types[type] then return \"ec2\" end\n if runtime_monitoring_finding_types[type] then return \"runtime\" end\n if iam_finding_types[type] then return \"iam\" end\n if kubernetes_audit_logs_finding_types[type] then return \"kubernetes\" end\n if lambda_protection_finding_types[type] then return \"lambda\" end\n if malware_protection_finding_types[type] then return \"malware\" end\n if rds_protection_finding_types[type] then return \"rds\" end\n if s3_finding_types[type] then return \"s3\" end\n return \"unknown\"\nend\n\n-- Helper: Encode Lua table to JSON string with field ordering\nlocal function encodeJson(obj, fieldOrder, key)\n if obj == nil then\n\t return \"null\"\n elseif type(obj) == \"boolean\" then\n\t return tostring(obj)\n elseif type(obj) == \"number\" then\n\t return tostring(obj)\n elseif type(obj) == \"string\" then\n\t return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n\t local isArray = true\n\t local maxIndex = 0\n\t for k, v in pairs(obj) do\n\t\t if type(k) ~= \"number\" then\n\t\t\t isArray = false\n\t\t\t break\n\t\t end\n\t\t maxIndex = math.max(maxIndex, k)\n\t end\n\t if isArray then\n\t\t local items = {}\n\t\t for i = 1, maxIndex do\n\t\t\t table.insert(items, encodeJson(obj[i], fieldOrder, key))\n\t\t end\n\t\t return \"[\" .. table.concat(items, \",\") .. \"]\"\n\t else\n\t\t local items = {}\n\t\t local fieldOrdering = fieldOrder[key] or {}\n\t\t \n\t\t -- Phase 1: ordered keys\n\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t local v = obj[fieldName]\n\t\t\t if v ~= nil then\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, fieldName)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t -- Phase 2: remaining keys (not in fieldOrder)\n\t\t for k, v in pairs(obj) do\n\t\t\t local found = false\n\t\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t\t if k == fieldName then\n\t\t\t\t\t found = true\n\t\t\t\t\t break\n\t\t\t\t end\n\t\t\t end\n\t\t\t if not found then\n\t\t\t\t local keyStr = type(k) == \"string\" and k or tostring(k)\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, keyStr)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t return \"{\" .. table.concat(items, \",\") .. \"}\"\n\t end\n else\n\t return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal function convertUtcToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\nlocal IGNORE_KEYS = {\n _ob = true,\n timestamp = true, -- Ignore timestamp as we use start_time/end_time\n}\n\n-- Global entry point expected by transform runners\nfunction processEvent(event)\n local site_id = nil\n local site_tbl = event and event[\"site\"]\n if type(site_tbl) == \"table\" and site_tbl[\"id\"] then\n site_id = site_tbl[\"id\"]\n elseif event and event[\"site_id\"] then\n site_id = event[\"site_id\"]\n end\n local fam = detect_family(event or {})\n local copy = deepCopy(event, IGNORE_KEYS)\n local result = processGuardDutyEvent(copy, site_id, fam)\n result.time = convertUtcToMilliseconds(copy[\"createdAt\"])\n result.message = encodeJson(copy, get_msg_field_ordering(fam), \"message\")\n return result\nend\n\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "F", - "harness_score": 45, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_guardduty/metadata.yaml b/pipelines/community/transform_ocsf/aws_guardduty/metadata.yaml deleted file mode 100644 index 92883b6..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/metadata.yaml +++ /dev/null @@ -1,51 +0,0 @@ -grade: - letter: F - score: 45 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Aws Guardduty. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: aws - dataSource: Aws Guardduty - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "Other - {Explain: AWS EventBridge or S3 export of GuardDuty findings}" - auth_type: "IAM Role" - sample_record: "{\n \"schemaVersion\": \"2.0\",\n \"accountId\": \"222708836859\",\n \"region\":\ - \ \"us-east-1\",\n \"partition\": \"aws\",\n \"id\": \"8db850b8-f1b1-4dfd-8676-efd7b4e8ee85\",\n\ - \ \"arn\": \"arn:aws:guardduty:us-east-1::84378c5c8013403891eb51ada1b2a47b:detector/84378c5c8013403891eb51ada1b2a47b/finding/8db850b8-f1b1-4dfd-8676-efd7b4e8ee85\"\ - ,\n \"type\": \"UnauthorizedAccess:IAMUser/ConsoleLogin\",\n \"title\": \"EC2 instance attempting\ - \ connection to a blackholed IP address.\",\n \"description\": \"EC2 instance is attempting to communicate\ - \ with a blackholed IP on port 80.\",\n \"severity\": 8.0,\n \"createdAt\": \"2026-04-20T03:40:52Z\"\ - ,\n \"updatedAt\": \"2026-04-20T03:40:52Z\",\n \"resource\": {\n \"resourceType\": \"Instance\"\ - ,\n \"instanceDetails\": {\n \"instanceId\": \"i-5c365f7b\",\n \"instanceType\": \"m5.large\"\ - ,\n \"platform\": null,\n \"networkInterfaces\": [\n {\n \"networkInterfaceId\"\ - : \"eni-26b50034\",\n \"privateIpAddress\": \"219.13.151.53\",\n \"publicIp\": \"\ - 19.41.200.229\",\n \"ipv6Addresses\": []\n }\n ],\n \"tags\": []\n }\n\ - \ },\n \"service\": {\n \"serviceName\": \"guardduty\",\n \"detectorId\": \"84378c5c8013403891eb51ada1b2a47b\"\ - ,\n \"eventFirstSeen\": \"2026-04-20T03:40:52Z\",\n \"eventLastSeen\": \"2026-04-20T03:40:52Z\"\ - ,\n \"count\": 1,\n \"action\": {\n \"actionType\": \"NETWORK_CONNECTION\",\n \"networkConnectionAction\"\ - : {\n \"connectionDirection\": \"OUTBOUND\",\n \"protocol\": \"TCP\",\n \"port\"\ - : 80,\n \"remoteIpDetails\": {\n \"ipAddressV4\": \"19" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: aws, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: F - harness_score: 45 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/aws_guardduty/sample.json b/pipelines/community/transform_ocsf/aws_guardduty/sample.json deleted file mode 100644 index 6c70780..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/sample.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "schemaVersion": "2.0", - "accountId": "222708836859", - "region": "us-east-1", - "partition": "aws", - "id": "8db850b8-f1b1-4dfd-8676-efd7b4e8ee85", - "arn": "arn:aws:guardduty:us-east-1::84378c5c8013403891eb51ada1b2a47b:detector/84378c5c8013403891eb51ada1b2a47b/finding/8db850b8-f1b1-4dfd-8676-efd7b4e8ee85", - "type": "UnauthorizedAccess:IAMUser/ConsoleLogin", - "title": "EC2 instance attempting connection to a blackholed IP address.", - "description": "EC2 instance is attempting to communicate with a blackholed IP on port 80.", - "severity": 8.0, - "createdAt": "2026-04-20T03:40:52Z", - "updatedAt": "2026-04-20T03:40:52Z", - "resource": { - "resourceType": "Instance", - "instanceDetails": { - "instanceId": "i-5c365f7b", - "instanceType": "m5.large", - "platform": null, - "networkInterfaces": [ - { - "networkInterfaceId": "eni-26b50034", - "privateIpAddress": "219.13.151.53", - "publicIp": "19.41.200.229", - "ipv6Addresses": [] - } - ], - "tags": [] - } - }, - "service": { - "serviceName": "guardduty", - "detectorId": "84378c5c8013403891eb51ada1b2a47b", - "eventFirstSeen": "2026-04-20T03:40:52Z", - "eventLastSeen": "2026-04-20T03:40:52Z", - "count": 1, - "action": { - "actionType": "NETWORK_CONNECTION", - "networkConnectionAction": { - "connectionDirection": "OUTBOUND", - "protocol": "TCP", - "port": 80, - "remoteIpDetails": { - "ipAddressV4": "190.60.202.107", - "organization": { - "asn": "-1", - "asnOrg": "GeneratedASNOrg", - "isp": "GeneratedISP", - "org": "GeneratedORG" - } - } - } - } - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_guardduty/serializer.lua b/pipelines/community/transform_ocsf/aws_guardduty/serializer.lua deleted file mode 100644 index f715edd..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/serializer.lua +++ /dev/null @@ -1,1720 +0,0 @@ --- AWS GuardDuty OCSF 1.0.0 parser (ported from Python) - --- Helper: split string by delimiter -local function split(str, delimiter) - local result = {} - local escaped = delimiter:gsub("[%.%+%*%?%^%$%(%)%[%]%%]", "%%%1") - local pattern = "([^" .. escaped .. "]+)" - for token in tostring(str):gmatch(pattern) do - table.insert(result, token) - end - if #result == 0 and #tostring(str) > 0 then - table.insert(result, str) - end - return result -end - --- Helper: safely access nested keys: keys is array, obj is table -local function getByPath(obj, keys) - local current = obj - for _, k in ipairs(keys) do - if current ~= nil and type(current) == "table" then - current = current[k] - else - return nil - end - end - return current -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - --- Common mapping applied before specific finders -local function common_mapping(event, site_id) - event["dataSource"] = { category = "security", name = "AWS GuardDuty", vendor = "AWS" } - event["metadata"] = { version = "1.0.0", product = { name = "AWS GuardDuty", vendor_name = "AWS" } } - event["cloudProvider"] = "AWS" - event["cloudAccountType"] = "AWS Account" - event["accountTypeId"] = 10 - if site_id ~= nil and site_id ~= "" then - event["site"] = { id = site_id } - end - local sev = event["severity"] - if type(sev) == "number" then - if sev >= 1.0 and sev <= 2.9 then - event["severity_id"] = 2 - elseif sev >= 4.0 and sev <= 6.9 then - event["severity_id"] = 3 - elseif sev >= 7.0 and sev <= 8.9 then - event["severity_id"] = 4 - else - event["severity_id"] = 0 - end - end -end - --- After mapping, convert lat/lon into coordinates and remove lat/lon fields based on activity_name -local function coordinates_mapping(parsed) - local lat = parsed["lat"] - local lon = parsed["lon"] - if (type(lat) == "number" or type(lat) == "string") or (type(lon) == "number" or type(lon) == "string") then - local latnum = tonumber(lat) - local lonnum = tonumber(lon) - if latnum and lonnum then - local coords = { latnum, lonnum } - local activity = parsed["activity_name"] - if activity == "EC2 finding types" or - activity == "EKS Runtime Monitoring finding types" or - activity == "Kubernetes Audit Logs finding types" or - activity == "RDS Protection finding types" or - activity == "S3 finding types" then - parsed["src_endpoint.location.coordinates"] = coords - parsed["lat"], parsed["lon"] = nil, nil - elseif activity == "IAM finding types" or activity == "Lambda Protection finding types" then - parsed["device.location.coordinates"] = coords - parsed["lat"], parsed["lon"] = nil, nil - end - end - end -end - -local function flatten_table(tbl, prefix, out) - out = out or {} - prefix = prefix or "" - local hasArrayElements = false - - for k, v in pairs(tbl) do - if type(k) == "number" then - hasArrayElements = true - break - end - end - - if hasArrayElements then - -- This is an array, store it as-is - if prefix ~= "" then - out[prefix] = tbl - end - return out - end - - for k, v in pairs(tbl) do - local key = prefix ~= "" and (prefix .. "." .. k) or k - if type(v) == "table" then - flatten_table(v, key, out) - else - out[key] = v - end - end - - return out -end - --- Apply a mapping table of { [source_dotted] = target_dotted } -local function apply_mapping(event, mapping) - local out = {} - for src, dst in pairs(mapping) do - local val = getByPath(event, split(src, ".")) - if val ~= nil then - out[dst] = val - end - end - - -- NEW: auto-capture unmapped fields - local copy = deepCopy(event) - local flattenCopy = flatten_table(copy) - for key, val in pairs(flattenCopy) do - if mapping[key] == nil and val ~= nil and val ~= "" and val ~= "[]" and val ~= "()" and val ~= "{}" then - out["unmapped." .. key] = val - end - end - return out -end - -local EC2_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition","id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "instanceDetails" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", - "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "networkConnectionAction" - }, - networkConnectionAction = { - "connectionDirection", "localIpDetails", "remoteIpDetails", "remotePortDetails", - "localPortDetails", "protocol", "blocked" - }, - localIpDetails = { - "ipAddressV4" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - remotePortDetails = { - "port", "portName" - }, - localPortDetails = { - "port", "portName" - }, - additionalInfo = { - "threatListName", "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local EKS_RUNTIME_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "eksClusterDetails", "kubernetesDetails", "containerDetails", "instanceDetails" - }, - eksClusterDetails = { - "name", "arn", "createdAt", "vpcId", "status", "tags" - }, - tags = { - "key", "value" - }, - kubernetesDetails = { - "kubernetesWorkloadDetails" - }, - kubernetesWorkloadDetails = { - "name", "namespace", "type", "uid" - }, - containerDetails = { - "id", "name", "image" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - service = { - "serviceName", "detectorId", "action", "runtimeDetails", "featureName", "resourceRole", - "additionalInfo", "evidence", "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "dnsRequestAction" - }, - dnsRequestAction = { - "domain", "protocol", "blocked", "domainWithSuffix" - }, - runtimeDetails = { - "process" - }, - process = { - "pid", "name", "uuid", "executablePath", "executableSha256", "cmdLine", "user", "euid", - "userId", "pwd", "startTime", "parentUuid", "lineage" - }, - lineage = { - "pid", "uuid", "executablePath", "euid", "parentUuid" - }, - additionalInfo = { - "threatListName", "sample", "agentDetails", "value", "type" - }, - agentDetails = { - "agentVersion", "agentId" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local IAM_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "accessKeyDetails", "instanceDetails" - }, - accessKeyDetails = { - "accessKeyId", "principalId", "userType", "userName" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "awsApiCallAction" - }, - awsApiCallAction = { - "api", "serviceName", "callerType", "errorCode", "remoteIpDetails", "affectedResources" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "userAgent", "anomalies", "profiledBehavior", "unusualBehavior", "sample", "value", "type" - }, - userAgent = { - "fullUserAgent", "userAgentCategory" - }, - anomalies = { - "anomalousAPIs" - }, - profiledBehavior = { - "rareProfiledAPIsAccountProfiling", "infrequentProfiledAPIsAccountProfiling", - "frequentProfiledAPIsAccountProfiling", "rareProfiledAPIsUserIdentityProfiling", - "infrequentProfiledAPIsUserIdentityProfiling", "frequentProfiledAPIsUserIdentityProfiling", - "rareProfiledUserTypesAccountProfiling", "infrequentProfiledUserTypesAccountProfiling", - "frequentProfiledUserTypesAccountProfiling", "rareProfiledUserNamesAccountProfiling", - "infrequentProfiledUserNamesAccountProfiling", "frequentProfiledUserNamesAccountProfiling", - "rareProfiledASNsAccountProfiling", "infrequentProfiledASNsAccountProfiling", - "frequentProfiledASNsAccountProfiling", "rareProfiledASNsUserIdentityProfiling", - "infrequentProfiledASNsUserIdentityProfiling", "frequentProfiledASNsUserIdentityProfiling", - "rareProfiledUserAgentsAccountProfiling", "infrequentProfiledUserAgentsAccountProfiling", - "frequentProfiledUserAgentsAccountProfiling", "rareProfiledUserAgentsUserIdentityProfiling", - "infrequentProfiledUserAgentsUserIdentityProfiling", "frequentProfiledUserAgentsUserIdentityProfiling" - }, - unusualBehavior = { - "unusualAPIsAccountProfiling", "unusualAPIsUserIdentityProfiling", - "unusualUserTypesAccountProfiling", "unusualUserNamesAccountProfiling", - "unusualASNsAccountProfiling", "unusualASNsUserIdentityProfiling", - "unusualUserAgentsAccountProfiling", "unusualUserAgentsUserIdentityProfiling", - "isUnusualUserIdentity" - }, -} - -local KUBERNETES_AUDIT_LOGS_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "eksClusterDetails", "kubernetesDetails", "accessKeyDetails" - }, - eksClusterDetails = { - "name", "arn", "createdAt", "vpcId", "status", "tags" - }, - tags = { - "key", "value" - }, - kubernetesDetails = { - "kubernetesWorkloadDetails", "kubernetesUserDetails" - }, - kubernetesWorkloadDetails = { - "name", "namespace", "type", "uid" - }, - kubernetesUserDetails = { - "username", "uid", "groups", "sessionName" - }, - accessKeyDetails = { - "accessKeyId", "principalId", "userType", "userName" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "kubernetesApiCallAction" - }, - kubernetesApiCallAction = { - "requestUri", "verb", "sourceIPs", "userAgent", "remoteIpDetails", "statusCode" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "threatName", "threatListName", "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local LAMBDA_PROTECTION_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "lambdaDetails" - }, - lambdaDetails = { - "functionArn", "functionName", "description", "lastModifiedAt", "revisionId", - "functionVersion", "role", "vpcConfig", "tags" - }, - vpcConfig = { - "vpcId", "securityGroups", "subnetIds" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "networkConnectionAction" - }, - networkConnectionAction = { - "connectionDirection", "remoteIpDetails", "remotePortDetails", "protocol", "blocked" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - remotePortDetails = { - "port", "portName" - }, - additionalInfo = { - "unusualProtocol", "threatListName", "unusual", "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local MALWARE_PROTECTION_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "instanceDetails", "ebsVolumeDetails" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - ebsVolumeDetails = { - "scannedVolumeDetails", "skippedVolumeDetails" - }, - scannedVolumeDetails = { - "volumeArn", "volumeType", "deviceName", "volumeSizeInGB", "encryptionType", "snapshotArn", "kmsKeyArn" - }, - service = { - "serviceName", "detectorId", "featureName", "ebsVolumeScanDetails", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - ebsVolumeScanDetails = { - "scanId", "scanStartedAt", "scanCompletedAt", "scanType", "triggerFindingId", "sources", "scanDetections" - }, - scanDetections = { - "scannedItemCount", "threatsDetectedItemCount", "highestSeverityThreatDetails", "threatDetectedByName" - }, - scannedItemCount = { - "totalGb", "files", "volumes" - }, - threatsDetectedItemCount = { - "files" - }, - highestSeverityThreatDetails = { - "severity", "threatName", "count" - }, - threatDetectedByName = { - "itemCount", "uniqueThreatNameCount", "shortened", "threatNames" - }, - threatNames = { - "name", "severity", "itemCount", "filePaths" - }, - filePaths = { - "filePath", "fileName", "volumeArn", "hash" - }, - additionalInfo = { - "sample", "value", "type" - }, -} - -local RDS_PROTECTION_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "rdsDbInstanceDetails", "rdsDbUserDetails" - }, - rdsDbInstanceDetails = { - "dbInstanceIdentifier", "engine", "engineVersion", "dbClusterIdentifier", "dbInstanceArn", "tags" - }, - tags = { - "key", "value" - }, - rdsDbUserDetails = { - "user", "application", "database", "ssl", "authMethod" - }, - service = { - "action", "additionalInfo", "resourceRole", "evidence", "count", "detectorId", - "eventFirstSeen", "eventLastSeen", "serviceName", "archived" - }, - action = { - "actionType", "rdsLoginAttemptAction" - }, - rdsLoginAttemptAction = { - "remoteIpDetails" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local S3_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "accessKeyDetails", "s3BucketDetails", "instanceDetails" - }, - accessKeyDetails = { - "accessKeyId", "principalId", "userType", "userName" - }, - s3BucketDetails = { - "arn", "name", "type", "createdAt", "owner", "tags", "defaultServerSideEncryption", "publicAccess" - }, - owner = { - "id" - }, - tags = { - "key", "value" - }, - defaultServerSideEncryption = { - "encryptionType", "kmsMasterKeyArn" - }, - publicAccess = { - "permissionConfiguration", "effectivePermission" - }, - permissionConfiguration = { - "bucketLevelPermissions", "accountLevelPermissions" - }, - bucketLevelPermissions = { - "accessControlList", "bucketPolicy", "blockPublicAccess" - }, - accessControlList = { - "allowsPublicReadAccess", "allowsPublicWriteAccess" - }, - bucketPolicy = { - "allowsPublicReadAccess", "allowsPublicWriteAccess" - }, - blockPublicAccess = { - "ignorePublicAcls", "restrictPublicBuckets", "blockPublicAcls", "blockPublicPolicy" - }, - accountLevelPermissions = { - "blockPublicAccess" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "awsApiCallAction" - }, - awsApiCallAction = { - "api", "serviceName", "callerType", "errorCode", "remoteIpDetails", "affectedResources" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "unusual", "sample", "value", "type" - }, - unusual = { - "hoursOfDay", "userNames" - }, -} - -local function get_msg_field_ordering(family) - if family == "ec2" then return EC2_FIELD_ORDER end - if family == "runtime" or family == "eks_runtime" or family == "runtime_monitoring" then return EKS_RUNTIME_FIELD_ORDER end - if family == "iam" then return IAM_FIELD_ORDER end - if family == "kubernetes" or family == "k8s" then return KUBERNETES_AUDIT_LOGS_FIELD_ORDER end - if family == "lambda" then return LAMBDA_PROTECTION_FIELD_ORDER end - if family == "malware" then return MALWARE_PROTECTION_FIELD_ORDER end - if family == "rds" then return RDS_PROTECTION_FIELD_ORDER end - if family == "s3" then return S3_FIELD_ORDER end - return {} -end - --- OCSF Mappings (ported 1:1 from Python) -local function ec2_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.instanceDetails.instanceId"] = "device.instance_uid", - ["resource.instanceDetails.networkInterfaces"] = "device.network_interfaces", - ["resource.instanceDetails.imageId"] = "device.image.uid", - ["service.serviceName"] = "src_endpoint.svc_name", - ["service.detectorId"] = "src_endpoint.uid", - ["service.action.networkConnectionAction.localIpDetails.ipAddressV4"] = "src_endpoint.ip", - ["service.action.networkConnectionAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.networkConnectionAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.networkConnectionAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.action.networkConnectionAction.localPortDetails.port"] = "src_endpoint.port", - ["service.action.networkConnectionAction.protocol"] = "connection_info.protocol_name", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function runtime_monitoring_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.instanceDetails.instanceId"] = "device.instance_uid", - ["resource.instanceDetails.networkInterfaces"] = "device.network_interfaces", - ["resource.instanceDetails.imageId"] = "device.image.uid", - ["service.runtimeDetails.process.pid"] = "actor.process.uid", - ["service.runtimeDetails.process.name"] = "actor.process.name", - ["service.runtimeDetails.process.user"] = "actor.user.name", - ["service.runtimeDetails.process.userId"] = "actor.user.uid", - ["service.runtimeDetails.process.parentUuid"] = "actor.process.parent_process.uid", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.featureName"] = "api.service.name", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function iam_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.accessKeyDetails.accessKeyId"] = "actor.session.credential_uid", - ["resource.accessKeyDetails.principalId"] = "actor.session.uid", - ["resource.accessKeyDetails.userName"] = "actor.user.name", - ["resource.instanceDetails.instanceId"] = "device.instance_uid", - ["resource.instanceDetails.networkInterfaces"] = "device.network_interfaces", - ["resource.instanceDetails.imageId"] = "device.image.uid", - ["service.action.awsApiCallAction.serviceName"] = "api.service.name", - ["service.action.awsApiCallAction.remoteIpDetails.organization.isp"] = "device.location.isp", - ["service.action.awsApiCallAction.remoteIpDetails.country.countryName"] = "device.location.country", - ["service.action.awsApiCallAction.remoteIpDetails.city.cityName"] = "device.location.city", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function kubernetes_audit_log_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.kubernetesDetails.kubernetesUserDetails.username"] = "actor.user.name", - ["resource.kubernetesDetails.kubernetesUserDetails.uid"] = "actor.user.uid", - ["resource.kubernetesDetails.kubernetesUserDetails.groups"] = "actor.user.groups", - ["resource.accessKeyDetails.accessKeyId"] = "actor.session.credential_uid", - ["resource.accessKeyDetails.principalId"] = "actor.session.uid", - ["service.serviceName"] = "src_endpoint.svc_name", - ["service.detectorId"] = "src_endpoint.uid", - ["service.action.kubernetesApiCallAction.requestUri"] = "http_request.url.url_string", - ["service.action.kubernetesApiCallAction.verb"] = "http_request.http_method", - ["service.action.kubernetesApiCallAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.kubernetesApiCallAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.kubernetesApiCallAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.action.kubernetesApiCallAction.statusCode"] = "status_code", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function lambda_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["service.action.networkConnectionAction.remoteIpDetails.organization.isp"] = "device.location.isp", - ["service.action.networkConnectionAction.remoteIpDetails.country.countryName"] = "device.location.country", - ["service.action.networkConnectionAction.remoteIpDetails.city.cityName"] = "device.location.city", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function malware_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["partition"] = "resources.cloud_partition", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["service.featureName"] = "api.service.name", - ["service.ebsVolumeScanDetails.scanId"] = "api.service.uid", - ["service.ebsVolumeScanDetails.scanStartedAt"] = "finding.first_seen_time", - ["service.ebsVolumeScanDetails.scanCompletedAt"] = "finding.last_seen_time", - ["service.ebsVolumeScanDetails.triggerFindingId"] = "finding.uid", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function rds_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.detectorId"] = "src_endpoint.uid", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["service.serviceName"] = "src_endpoint.svc_name", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function s3_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["partition"] = "resources.cloud_partition", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.accessKeyDetails.accessKeyId"] = "resources.user.credential_uid", - ["resource.accessKeyDetails.userType"] = "resources.user.type", - ["resource.accessKeyDetails.userName"] = "resources.user.name", - ["service.serviceName"] = "src_endpoint.svc_name", - ["service.detectorId"] = "src_endpoint.uid", - ["service.action.awsApiCallAction.serviceName"] = "api.service.name", - ["service.action.awsApiCallAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.awsApiCallAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.awsApiCallAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function unknown_findings_ocsf_mapping() - return { - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - --- Finding types constants -local ec2_finding_types = { - ["Backdoor:EC2/C&CActivity.B"] = true, - ["Backdoor:EC2/C&CActivity.B!DNS"] = true, - ["Backdoor:EC2/DenialOfService.Dns"] = true, - ["Backdoor:EC2/DenialOfService.Tcp"] = true, - ["Backdoor:EC2/DenialOfService.Udp"] = true, - ["Backdoor:EC2/DenialOfService.UdpOnTcpPorts"] = true, - ["Backdoor:EC2/DenialOfService.UnusualProtocol"] = true, - ["Backdoor:EC2/Spambot"] = true, - ["Behavior:EC2/NetworkPortUnusual"] = true, - ["Behavior:EC2/TrafficVolumeUnusual"] = true, - ["CryptoCurrency:EC2/BitcoinTool.B"] = true, - ["CryptoCurrency:EC2/BitcoinTool.B!DNS"] = true, - ["DefenseEvasion:EC2/UnusualDNSResolver"] = true, - ["DefenseEvasion:EC2/UnusualDoHActivity"] = true, - ["DefenseEvasion:EC2/UnusualDoTActivity"] = true, - ["Impact:EC2/AbusedDomainRequest.Reputation"] = true, - ["Impact:EC2/BitcoinDomainRequest.Reputation"] = true, - ["Impact:EC2/MaliciousDomainRequest.Reputation"] = true, - ["Impact:EC2/PortSweep"] = true, - ["Impact:EC2/SuspiciousDomainRequest.Reputation"] = true, - ["Impact:EC2/WinRMBruteForce"] = true, - ["Recon:EC2/PortProbeEMRUnprotectedPort"] = true, - ["Recon:EC2/PortProbeUnprotectedPort"] = true, - ["Recon:EC2/Portscan"] = true, - ["Trojan:EC2/BlackholeTraffic"] = true, - ["Trojan:EC2/BlackholeTraffic!DNS"] = true, - ["Trojan:EC2/DGADomainRequest.B"] = true, - ["Trojan:EC2/DGADomainRequest.C!DNS"] = true, - ["Trojan:EC2/DNSDataExfiltration"] = true, - ["Trojan:EC2/DriveBySourceTraffic!DNS"] = true, - ["Trojan:EC2/DropPoint"] = true, - ["Trojan:EC2/DropPoint!DNS"] = true, - ["Trojan:EC2/PhishingDomainRequest!DNS"] = true, - ["UnauthorizedAccess:EC2/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:EC2/MetadataDNSRebind"] = true, - ["UnauthorizedAccess:EC2/RDPBruteForce"] = true, - ["UnauthorizedAccess:EC2/SSHBruteForce"] = true, - ["UnauthorizedAccess:EC2/TorClient"] = true, - ["UnauthorizedAccess:EC2/TorRelay"] = true -} - -local runtime_monitoring_finding_types = { - ["CryptoCurrency:Runtime/BitcoinTool.B"] = true, - ["Backdoor:Runtime/C&CActivity.B"] = true, - ["UnauthorizedAccess:Runtime/TorRelay"] = true, - ["UnauthorizedAccess:Runtime/TorClient"] = true, - ["Trojan:Runtime/BlackholeTraffic"] = true, - ["Trojan:Runtime/DropPoint"] = true, - ["CryptoCurrency:Runtime/BitcoinTool.B!DNS"] = true, - ["Backdoor:Runtime/C&CActivity.B!DNS"] = true, - ["Trojan:Runtime/BlackholeTraffic!DNS"] = true, - ["Trojan:Runtime/DropPoint!DNS"] = true, - ["Trojan:Runtime/DGADomainRequest.C!DNS"] = true, - ["Trojan:Runtime/DriveBySourceTraffic!DNS"] = true, - ["Trojan:Runtime/PhishingDomainRequest!DNS"] = true, - ["Impact:Runtime/AbusedDomainRequest.Reputation"] = true, - ["Impact:Runtime/BitcoinDomainRequest.Reputation"] = true, - ["Impact:Runtime/MaliciousDomainRequest.Reputation"] = true, - ["Impact:Runtime/SuspiciousDomainRequest.Reputation"] = true, - ["UnauthorizedAccess:Runtime/MetadataDNSRebind"] = true, - ["Execution:Runtime/NewBinaryExecuted"] = true, - ["PrivilegeEscalation:Runtime/DockerSocketAccessed"] = true, - ["PrivilegeEscalation:Runtime/RuncContainerEscape"] = true, - ["PrivilegeEscalation:Runtime/CGroupsReleaseAgentModified"] = true, - ["DefenseEvasion:Runtime/ProcessInjection.Proc"] = true, - ["DefenseEvasion:Runtime/ProcessInjection.Ptrace"] = true, - ["DefenseEvasion:Runtime/ProcessInjection.VirtualMemoryWrite"] = true, - ["Execution:Runtime/ReverseShell"] = true, - ["DefenseEvasion:Runtime/FilelessExecution"] = true, - ["Impact:Runtime/CryptoMinerExecuted"] = true, - ["Execution:Runtime/NewLibraryLoaded"] = true, - ["PrivilegeEscalation:Runtime/ContainerMountsHostDirectory"] = true, - ["PrivilegeEscalation:Runtime/UserfaultfdUsage"] = true -} - -local iam_finding_types = { - ["CredentialAccess:IAMUser/AnomalousBehavior"] = true, - ["DefenseEvasion:IAMUser/AnomalousBehavior"] = true, - ["Discovery:IAMUser/AnomalousBehavior"] = true, - ["Exfiltration:IAMUser/AnomalousBehavior"] = true, - ["Impact:IAMUser/AnomalousBehavior"] = true, - ["InitialAccess:IAMUser/AnomalousBehavior"] = true, - ["PenTest:IAMUser/KaliLinux"] = true, - ["PenTest:IAMUser/ParrotLinux"] = true, - ["PenTest:IAMUser/PentooLinux"] = true, - ["Persistence:IAMUser/AnomalousBehavior"] = true, - ["Policy:IAMUser/RootCredentialUsage"] = true, - ["PrivilegeEscalation:IAMUser/AnomalousBehavior"] = true, - ["Recon:IAMUser/MaliciousIPCaller"] = true, - ["Recon:IAMUser/MaliciousIPCaller.Custom"] = true, - ["Recon:IAMUser/TorIPCaller"] = true, - ["Stealth:IAMUser/CloudTrailLoggingDisabled"] = true, - ["Stealth:IAMUser/PasswordPolicyChange"] = true, - ["UnauthorizedAccess:IAMUser/ConsoleLoginSuccess.B"] = true, - ["UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.InsideAWS"] = true, - ["UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.OutsideAWS"] = true, - ["UnauthorizedAccess:IAMUser/MaliciousIPCaller"] = true, - ["UnauthorizedAccess:IAMUser/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:IAMUser/TorIPCaller"] = true -} - -local kubernetes_audit_logs_finding_types = { - ["CredentialAccess:Kubernetes/MaliciousIPCaller"] = true, - ["CredentialAccess:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["CredentialAccess:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["CredentialAccess:Kubernetes/TorIPCaller"] = true, - ["DefenseEvasion:Kubernetes/MaliciousIPCaller"] = true, - ["DefenseEvasion:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["DefenseEvasion:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["DefenseEvasion:Kubernetes/TorIPCaller"] = true, - ["Discovery:Kubernetes/MaliciousIPCaller"] = true, - ["Discovery:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["Discovery:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["Discovery:Kubernetes/TorIPCaller"] = true, - ["Execution:Kubernetes/ExecInKubeSystemPod"] = true, - ["Impact:Kubernetes/MaliciousIPCaller"] = true, - ["Impact:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["Impact:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["Impact:Kubernetes/TorIPCaller"] = true, - ["Persistence:Kubernetes/ContainerWithSensitiveMount"] = true, - ["Persistence:Kubernetes/MaliciousIPCaller"] = true, - ["Persistence:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["Persistence:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["Persistence:Kubernetes/TorIPCaller"] = true, - ["Policy:Kubernetes/AdminAccessToDefaultServiceAccount"] = true, - ["Policy:Kubernetes/AnonymousAccessGranted"] = true, - ["Policy:Kubernetes/ExposedDashboard"] = true, - ["Policy:Kubernetes/KubeflowDashboardExposed"] = true, - ["PrivilegeEscalation:Kubernetes/PrivilegedContainer"] = true, - ["CredentialAccess:Kubernetes/AnomalousBehavior.SecretsAccessed"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleBindingCreated"] = true, - ["Execution:Kubernetes/AnomalousBehavior.ExecInPod"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!PrivilegedContainer"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!ContainerWithSensitiveMount"] = true, - ["Execution:Kubernetes/AnomalousBehavior.WorkloadDeployed"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleCreated"] = true, - ["Discovery:Kubernetes/AnomalousBehavior.PermissionChecked"] = true -} - -local lambda_protection_finding_types = { - ["Backdoor:Lambda/C&CActivity.B"] = true, - ["CryptoCurrency:Lambda/BitcoinTool.B"] = true, - ["Trojan:Lambda/BlackholeTraffic"] = true, - ["Trojan:Lambda/DropPoint"] = true, - ["UnauthorizedAccess:Lambda/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:Lambda/TorClient"] = true, - ["UnauthorizedAccess:Lambda/TorRelay"] = true -} - -local malware_protection_finding_types = { - ["Execution:EC2/MaliciousFile"] = true, - ["Execution:ECS/MaliciousFile"] = true, - ["Execution:Kubernetes/MaliciousFile"] = true, - ["Execution:Container/MaliciousFile"] = true, - ["Execution:EC2/SuspiciousFile"] = true, - ["Execution:ECS/SuspiciousFile"] = true, - ["Execution:Kubernetes/SuspiciousFile"] = true, - ["Execution:Container/SuspiciousFile"] = true -} - -local rds_protection_finding_types = { - ["CredentialAccess:RDS/AnomalousBehavior.SuccessfulLogin"] = true, - ["CredentialAccess:RDS/AnomalousBehavior.FailedLogin"] = true, - ["CredentialAccess:RDS/AnomalousBehavior.SuccessfulBruteForce"] = true, - ["CredentialAccess:RDS/MaliciousIPCaller.SuccessfulLogin"] = true, - ["CredentialAccess:RDS/MaliciousIPCaller.FailedLogin"] = true, - ["Discovery:RDS/MaliciousIPCaller"] = true, - ["CredentialAccess:RDS/TorIPCaller.SuccessfulLogin"] = true, - ["CredentialAccess:RDS/TorIPCaller.FailedLogin"] = true, - ["Discovery:RDS/TorIPCaller"] = true -} - -local s3_finding_types = { - ["Discovery:S3/AnomalousBehavior"] = true, - ["Discovery:S3/MaliciousIPCaller"] = true, - ["Discovery:S3/MaliciousIPCaller.Custom"] = true, - ["Discovery:S3/TorIPCaller"] = true, - ["Exfiltration:S3/AnomalousBehavior"] = true, - ["Exfiltration:S3/MaliciousIPCaller"] = true, - ["Impact:S3/AnomalousBehavior.Delete"] = true, - ["Impact:S3/AnomalousBehavior.Permission"] = true, - ["Impact:S3/AnomalousBehavior.Write"] = true, - ["Impact:S3/MaliciousIPCaller"] = true, - ["PenTest:S3/KaliLinux"] = true, - ["PenTest:S3/ParrotLinux"] = true, - ["PenTest:S3/PentooLinux"] = true, - ["Policy:S3/AccountBlockPublicAccessDisabled"] = true, - ["Policy:S3/BucketAnonymousAccessGranted"] = true, - ["Policy:S3/BucketBlockPublicAccessDisabled"] = true, - ["Policy:S3/BucketPublicAccessGranted"] = true, - ["Stealth:S3/ServerAccessLoggingDisabled"] = true, - ["UnauthorizedAccess:S3/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:S3/TorIPCaller"] = true -} - - --- Per-category helpers (mirroring Python OCSFMapperHelper) -local function ec2_findings(event, site_id) - event["class_uid"] = 4001 - event["class_name"] = "Network Activity" - event["category_uid"] = 4 - event["category_name"] = "Network Activity" - event["activity_id"] = "99" - event["activity_name"] = "EC2 finding types" - event["type_uid"] = 400199 - event["type_name"] = "EC2 finding types" - event["eventId"] = "EC2 finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, ec2_findings_ocsf_mapping()) - -- enrich with fields that were set directly on event - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function runtime_monitoring_findings(event, site_id) - event["class_uid"] = 6003 - event["class_name"] = "API Activity" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "EKS Runtime Monitoring finding types" - event["type_uid"] = 600399 - event["type_name"] = "EKS Runtime Monitoring finding types" - event["eventId"] = "EKS Runtime Monitoring finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, runtime_monitoring_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function iam_findings(event, site_id) - event["class_uid"] = 3004 - event["class_name"] = "Entity Management" - event["category_uid"] = 3 - event["category_name"] = "Identity & Access Management" - event["activity_id"] = "99" - event["activity_name"] = "IAM finding types" - event["type_uid"] = 300499 - event["type_name"] = "IAM finding types" - event["eventId"] = "IAM finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, iam_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function kubernetes_audit_log_findings(event, site_id) - event["class_uid"] = 6003 - event["class_name"] = "API Activity" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "Kubernetes Audit Logs finding types" - event["type_uid"] = 600399 - event["type_name"] = "Kubernetes Audit Logs finding types" - event["eventId"] = "Kubernetes Audit Logs finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, kubernetes_audit_log_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function lambda_findings(event, site_id) - event["class_uid"] = 6002 - event["class_name"] = "Application Lifecycle" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "Lambda Protection finding types" - event["type_uid"] = 600299 - event["type_name"] = "Lambda Protection finding types" - event["eventId"] = "Lambda Protection finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, lambda_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function malware_findings(event, site_id) - event["class_uid"] = 2001 - event["class_name"] = "Security Finding" - event["category_uid"] = 2 - event["category_name"] = "Findings" - event["activity_id"] = "99" - event["activity_name"] = "Malware Protection finding types" - event["type_uid"] = 200199 - event["type_name"] = "Malware Protection finding types" - event["eventId"] = "Malware Protection finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, malware_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - return flat -end - -local function rds_findings(event, site_id) - event["class_uid"] = 3002 - event["class_name"] = "Authentication" - event["category_uid"] = 3 - event["category_name"] = "Identity & Access Management" - event["activity_id"] = "99" - event["activity_name"] = "RDS Protection finding types" - event["type_uid"] = 300299 - event["type_name"] = "RDS Protection finding types" - event["eventId"] = "RDS Protection finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, rds_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function s3_findings(event, site_id) - event["class_uid"] = 6003 - event["class_name"] = "API Activity" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "S3 finding types" - event["type_uid"] = 600399 - event["type_name"] = "S3 finding types" - event["eventId"] = "S3 finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, s3_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function unknown_findings(event, site_id) - event["class_uid"] = 0 - event["class_name"] = "Base Event" - event["category_uid"] = 0 - event["category_name"] = "Uncategorized" - event["activity_id"] = 0 - event["activity_name"] = "Unknown" - event["type_uid"] = 0 - event["type_name"] = "Unknown" - event["eventId"] = "Unknown" - common_mapping(event, site_id) - local flat = apply_mapping(event, unknown_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - return flat -end - --- Dispatcher. family one of: "ec2","runtime","iam","kubernetes","lambda","malware","rds","s3","unknown" -local function processGuardDutyEvent(event, site_id, family) - local fam = tostring(family or "unknown"):lower() - if fam == "ec2" then return ec2_findings(deepCopy(event), site_id) end - if fam == "runtime" or fam == "eks_runtime" or fam == "runtime_monitoring" then return runtime_monitoring_findings(deepCopy(event), site_id) end - if fam == "iam" then return iam_findings(deepCopy(event), site_id) end - if fam == "kubernetes" or fam == "k8s" then return kubernetes_audit_log_findings(deepCopy(event), site_id) end - if fam == "lambda" then return lambda_findings(deepCopy(event), site_id) end - if fam == "malware" then return malware_findings(deepCopy(event), site_id) end - if fam == "rds" then return rds_findings(deepCopy(event), site_id) end - if fam == "s3" then return s3_findings(deepCopy(event), site_id) end - return unknown_findings(deepCopy(event), site_id) -end - --- Best-effort automatic family detection from GuardDuty event structure -local function detect_family(event) - local type = getByPath(event, {"type"}) - if ec2_finding_types[type] then return "ec2" end - if runtime_monitoring_finding_types[type] then return "runtime" end - if iam_finding_types[type] then return "iam" end - if kubernetes_audit_logs_finding_types[type] then return "kubernetes" end - if lambda_protection_finding_types[type] then return "lambda" end - if malware_protection_finding_types[type] then return "malware" end - if rds_protection_finding_types[type] then return "rds" end - if s3_finding_types[type] then return "s3" end - return "unknown" -end - --- Helper: Encode Lua table to JSON string with field ordering -local function encodeJson(obj, fieldOrder, key) - if obj == nil then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray then - local items = {} - for i = 1, maxIndex do - table.insert(items, encodeJson(obj[i], fieldOrder, key)) - end - return "[" .. table.concat(items, ",") .. "]" - else - local items = {} - local fieldOrdering = fieldOrder[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrdering) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldOrder, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - -- Phase 2: remaining keys (not in fieldOrder) - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrdering) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, fieldOrder, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - return "{" .. table.concat(items, ",") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local function convertUtcToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - -local IGNORE_KEYS = { - _ob = true, - timestamp = true, -- Ignore timestamp as we use start_time/end_time -} - --- Global entry point expected by transform runners -function processEvent(event) - local site_id = nil - local site_tbl = event and event["site"] - if type(site_tbl) == "table" and site_tbl["id"] then - site_id = site_tbl["id"] - elseif event and event["site_id"] then - site_id = event["site_id"] - end - local fam = detect_family(event or {}) - local copy = deepCopy(event, IGNORE_KEYS) - local result = processGuardDutyEvent(copy, site_id, fam) - result.time = convertUtcToMilliseconds(copy["createdAt"]) - result.message = encodeJson(copy, get_msg_field_ordering(fam), "message") - return result -end - diff --git a/pipelines/community/transform_ocsf/darktrace/darktrace.json b/pipelines/community/transform_ocsf/darktrace/darktrace.json deleted file mode 100644 index d9bd808..0000000 --- a/pipelines/community/transform_ocsf/darktrace/darktrace.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Darktrace", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Darktrace. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "darktrace", - "source_name": "darktrace", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "darktrace-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- Darktrace 1.1.0 OCSF Serializer\n\nlocal FEATURES = {\n CLEANUP_EMPTY_NULL = true,\n}\n\nlocal COMMON_MAPPING = {\n {source = \"action\", target = \"action\"},\n {source = \"action_id\", target = \"action_id\"},\n {source = \"category_uid\", target = \"category_uid\"},\n {source = \"category_name\", target = \"category_name\"},\n {source = \"class_uid\", target = \"class_uid\"},\n {source = \"class_name\", target = \"class_name\"},\n {source = \"activity_id\", target = \"activity_id\"},\n {source = \"activity_name\", target = \"activity_name\"},\n {source = \"type_uid\", target = \"type_uid\"},\n {source = \"type_name\", target = \"type_name\"},\n {source = \"severity_id\", target = \"severity_id\"},\n {source = \"metadata.product.vendor_name\", target = \"metadata.product.vendor_name\"},\n {source = \"metadata.product.name\", target = \"metadata.product.name\"},\n {source = \"metadata.version\", target = \"metadata.version\"},\n {source = \"metadata.original_time\", target = \"metadata.original_time\"},\n {source = \"dataSource.category\", target = \"dataSource.category\"},\n {source = \"dataSource.name\", target = \"dataSource.name\"},\n {source = \"dataSource.vendor\", target = \"dataSource.vendor\"},\n {source = \"site.id\", target = \"site.id\"},\n {source = \"message\", target = \"message\"},\n {source = \"user.type_id\", target = \"user.type_id\"},\n {source = \"observables\", target = \"observables\"}\n}\n\nlocal GROUPS_LOGS_MAPPING = {\n {source = \"id\", target = \"metadata.uid\"},\n {source = \"mitreTactics\", target = \"attacks.tactics\"},\n {source = \"category\", target = \"severity\"},\n {source = \"start\", target = \"start_time\"},\n {source = \"start_time\", target = \"time\"},\n {source = \"end\", target = \"end_time\"},\n {source = \"finding_info.uid\", target = \"finding_info.uid\"},\n {source = \"finding_info.first_seen_time\", target = \"finding_info.first_seen_time\"},\n {source = \"finding_info.title\", target = \"finding_info.title\"},\n {source = \"finding_info.product_uid\", target = \"finding_info.product_uid\"},\n {source = \"finding_info.desc\", target = \"finding_info.desc\"}\n}\n\nlocal INCIDENTS_LOGS_MAPPING = {\n {source = \"createdAt\", target = \"finding_info.created_time\"},\n {source = \"mitreTactics\", target = \"attacks.tactics\"},\n {source = \"start_time\", target = \"start_time\"},\n {source = \"end_time\", target = \"end_time\"},\n {source = \"device.namespace_pid\", target = \"device.namespace_pid\"},\n {source = \"device.hostname\", target = \"device.hostname\"},\n {source = \"device.ip\", target = \"device.ip\"},\n {source = \"device.mac\", target = \"device.mac\"},\n {source = \"device.subnet\", target = \"device.subnet\"},\n {source = \"device.uid\", target = \"device.uid\"},\n {source = \"device.subnet_uid\", target = \"device.subnet_uid\"},\n {source = \"device.type_id\", target = \"device.type_id\"},\n {source = \"authorizations\", target = \"authorizations\"},\n {source = \"risk_score\", target = \"risk_score\"},\n {source = \"id\", target = \"finding_info.uid\"},\n {source = \"title\", target = \"finding_info.title\"},\n {source = \"finding_info.first_seen_time\", target = \"finding_info.first_seen_time\"},\n {source = \"finding_info.product_uid\", target = \"finding_info.product_uid\"},\n {source = \"summary\", target = \"finding_info.desc\"},\n {source = \"severity\", target = \"severity\"},\n {source = \"metadata.original_time\", target = \"time\"}\n}\n\nlocal MODEL_BREACHES_LOGS_MAPPING = {\n {source = \"authorizations\", target = \"authorizations\"},\n {source = \"time\", target = \"metadata.original_time\"},\n {source = \"creationTime\", target = \"finding_info.created_time\"},\n {source = \"resources\", target = \"resources\"},\n {source = \"resources_result\", target = \"resources_result\"},\n {source = \"severity\", target = \"severity\"},\n {source = \"device.uid\", target = \"device.uid\"},\n {source = \"device.type_id\", target = \"device.type_id\"}\n}\n\nlocal STATUS_LOGS_MAPPING = {\n {source = \"ipAddress\", target = \"device.ip\"},\n {source = \"hostname\", target = \"device.hostname\"},\n {source = \"uuid\", target = \"device.uid\"},\n {source = \"device.type_id\", target = \"device.type_id\"},\n {source = \"device.type\", target = \"device.type\"},\n {source = \"device.subnet_uid\", target = \"device.subnet_uid\"},\n {source = \"device.network_interfaces\", target = \"device.network_interfaces\"},\n {source = \"severity\", target = \"severity\"}\n}\n\nlocal function convert_to_epoch_milliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+) (%d+):(%d+)\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\n-- Constant Fields\n\nlocal function get_group_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"uid\"] = event[\"id\"],\n [\"original_time\"] = type(event[\"start\"]) == \"number\" and event[\"start\"] or convert_to_epoch_milliseconds(event[\"start\"] or \"\"),\n },\n [\"action\"] = \"Other\",\n [\"action_id\"] = 99,\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Create\",\n [\"event.type\"] = \"Create\",\n [\"category_name\"] = \"Findings\",\n [\"category_uid\"] = 2,\n [\"class_name\"] = \"Detection Finding\",\n [\"class_uid\"] = 2004,\n [\"type_name\"] = \"Detection Finding: Create\",\n [\"type_uid\"] = 200401,\n [\"severity_id\"] = 99,\n [\"time\"] = type(event[\"start\"]) == \"number\" and event[\"start\"] or convert_to_epoch_milliseconds(event[\"start\"] or \"\")\n }\n\n local incident_events = event[\"incidentEvents\"] or {}\n -- Check if list is not empty (Lua tables are 1-indexed)\n if #incident_events > 0 then\n local incident = incident_events[1]\n fields[\"finding_info\"] = {\n [\"uid\"] = incident[\"uuid\"],\n [\"first_seen_time\"] = incident[\"start\"],\n [\"title\"] = incident[\"title\"],\n [\"product_uid\"] = incident[\"triggerDid\"],\n }\n end\n\n local observables_array = {}\n for _, incident in ipairs(incident_events) do\n -- Resource UID observable object\n local uuid_val = incident[\"uuid\"] or \"\"\n if uuid_val ~= \"\" then\n table.insert(observables_array, {type_id = 10, type = \"Resource UID\", name = \"finding_info.uid\", value = uuid_val})\n end\n\n -- Process observable object\n local title_val = incident[\"title\"] or \"\"\n if title_val ~= \"\" then\n table.insert(observables_array, {type_id = 25, type = \"Process\", name = \"finding_info.title\", value = title_val})\n end\n end\n\n if #observables_array > 0 then\n fields[\"observables\"] = observables_array\n end\n\n return fields\nend\n\nlocal function get_incidents_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"original_time\"] = type(event[\"relatedBreaches\"][1][\"timestamp\"]) == \"number\" and event[\"relatedBreaches\"][1][\"timestamp\"] or convert_to_epoch_milliseconds(event[\"relatedBreaches\"][1][\"timestamp\"] or \"\")\n },\n [\"action\"] = \"Other\",\n [\"action_id\"] = 99,\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Create\",\n [\"event.type\"] = \"Create\",\n [\"category_name\"] = \"Findings\",\n [\"category_uid\"] = 2,\n [\"class_name\"] = \"Detection Finding\",\n [\"class_uid\"] = 2004,\n [\"type_name\"] = \"Detection Finding: Create\",\n [\"type_uid\"] = 200401,\n [\"severity_id\"] = 99,\n [\"time\"] = type(event[\"relatedBreaches\"][1][\"timestamp\"]) == \"number\" and event[\"relatedBreaches\"][1][\"timestamp\"] or convert_to_epoch_milliseconds(event[\"relatedBreaches\"][1][\"timestamp\"] or \"\")\n }\n\n local periods = event[\"periods\"] or {}\n if #periods > 0 then\n fields[\"start_time\"] = periods[1][\"start\"]\n fields[\"end_time\"] = periods[1][\"end\"]\n end\n\n local related_breaches = event[\"relatedBreaches\"] or {}\n if #related_breaches > 0 then\n fields[\"authorizations\"] = {\n {\n [\"policy\"] = {\n [\"name\"] = related_breaches[1][\"modelName\"],\n [\"uid\"] = related_breaches[1][\"pbid\"]\n }\n }\n }\n fields[\"risk_score\"] = related_breaches[1][\"threatScore\"]\n end\n\n local breach_devices = event[\"breachDevices\"] or {}\n if #breach_devices > 0 then\n fields[\"device\"] = {\n [\"namespace_pid\"] = breach_devices[1][\"identifier\"],\n [\"hostname\"] = breach_devices[1][\"hostname\"],\n [\"ip\"] = breach_devices[1][\"ip\"],\n [\"mac\"] = breach_devices[1][\"mac\"],\n [\"subnet\"] = breach_devices[1][\"subnet\"],\n [\"uid\"] = breach_devices[1][\"did\"],\n [\"subnet_uid\"] = breach_devices[1][\"sid\"],\n [\"type_id\"] = 0\n }\n end\n\n local observables_array = {}\n for _, incident in ipairs(breach_devices) do\n -- Hostname observable object\n local hostname_val = incident[\"hostname\"] or \"\"\n if hostname_val ~= \"\" then\n local hostname_obj = {}\n hostname_obj[\"type_id\"] = 1\n hostname_obj[\"type\"] = \"Hostname\"\n hostname_obj[\"name\"] = \"device.hostname\"\n hostname_obj[\"value\"] = hostname_val\n table.insert(observables_array, hostname_obj)\n end\n\n -- IP Address observable object\n local ip_val = incident[\"ip\"] or \"\"\n if ip_val ~= \"\" then\n local ip_obj = {}\n ip_obj[\"type_id\"] = 2\n ip_obj[\"type\"] = \"IP Address\"\n ip_obj[\"name\"] = \"device.ip\"\n ip_obj[\"value\"] = ip_val\n table.insert(observables_array, ip_obj)\n end\n\n -- MAC Address observable object\n local mac_val = incident[\"mac\"] or \"\"\n if mac_val ~= \"\" then\n local mac_obj = {}\n mac_obj[\"type_id\"] = 3\n mac_obj[\"type\"] = \"MAC Address\"\n mac_obj[\"name\"] = \"device.mac\"\n mac_obj[\"value\"] = mac_val\n table.insert(observables_array, mac_obj)\n end\n end\n\n if #observables_array > 0 then\n fields[\"observables\"] = observables_array\n end\n\n return fields\nend\n\nlocal function get_modelbreaches_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"original_time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\")\n },\n [\"action\"] = \"Other\",\n [\"action_id\"] = 99,\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Create\",\n [\"event.type\"] = \"Create\",\n [\"category_name\"] = \"Findings\",\n [\"category_uid\"] = 2,\n [\"class_name\"] = \"Detection Finding\",\n [\"class_uid\"] = 2004,\n [\"type_name\"] = \"Detection Finding: Create\",\n [\"type_uid\"] = 200401,\n [\"severity_id\"] = 99,\n [\"time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\")\n }\n\n fields[\"authorizations\"] = {\n {\n [\"policy\"] = {\n [\"uid\"] = event[\"pbid\"]\n }\n }\n }\n\n local model = event[\"model\"] or {}\n local model_then = model[\"then\"]\n -- In Lua, empty tables are true, so check if not nil.\n -- If 'then' might be an empty table {}, you might need next(model_then) check, \n -- but usually API responses omit keys or return nil if missing.\n if model_then then\n fields[\"resources\"] = {\n {\n [\"name\"] = model_then[\"name\"],\n [\"uid\"] = model_then[\"uuid\"],\n [\"version\"] = model_then[\"version\"],\n [\"data\"] = model_then\n }\n }\n end\n\n local model_now = model[\"now\"]\n -- Note: Original Python code had a bug: 'if model_then:' check for 'resources_result'\n -- I assumed you wanted 'if model_now:' here based on context.\n -- If you want strict translation of the bug, change 'model_now' to 'model_then' in the if condition.\n if model_now then \n fields[\"resources_result\"] = {\n {\n [\"name\"] = model_now[\"name\"],\n [\"uid\"] = model_now[\"uuid\"],\n [\"version\"] = model_now[\"version\"],\n [\"data\"] = model_now\n }\n }\n end\n\n local device = event[\"device\"] or {}\n local device_did = device[\"did\"]\n if device_did then\n fields[\"device\"] = {\n [\"uid\"] = device_did,\n [\"type_id\"] = 0\n }\n end\n\n return fields\nend\n\nlocal function get_status_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"original_time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\")\n },\n [\"device\"] = {\n [\"type_id\"] = 99,\n [\"type\"] = event[\"type\"],\n [\"hostname\"] = event[\"hostname\"],\n [\"ip\"] = event[\"ipAddress\"],\n [\"network_interfaces\"] = {\n {\n [\"ip\"] = event[\"networkInterfacesAddress_eth0\"],\n [\"type_id\"] = 1,\n [\"type\"] = \"Wired\"\n }\n }\n },\n [\"time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\"),\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Log\",\n [\"event.type\"] = \"Log\",\n [\"category_name\"] = \"Discovery\",\n [\"category_uid\"] = 5,\n [\"class_name\"] = \"Device Config State\",\n [\"class_uid\"] = 5002,\n [\"type_name\"] = \"Device Config State: Log\",\n [\"type_uid\"] = 500201,\n [\"severity_id\"] = 99,\n }\n \n -- Initialize device table\n\n local subnet_data = event[\"subnetData\"] or {}\n if #subnet_data >= 1 then\n local sid = subnet_data[1][\"sid\"]\n fields[\"device\"][\"subnet_uid\"] = sid\n end\n\n fields[\"observables\"] = {}\n \n local hostname = event[\"hostname\"]\n if hostname and hostname ~= \"\" then\n table.insert(fields[\"observables\"], {\n [\"type_id\"] = 1,\n [\"type\"] = \"Hostname\",\n [\"name\"] = \"device.hostname\",\n [\"value\"] = hostname,\n })\n end\n\n local ip_address = event[\"ipAddress\"]\n if ip_address and ip_address ~= \"\" then\n table.insert(fields[\"observables\"], {\n [\"type_id\"] = 2,\n [\"type\"] = \"IP Address\",\n [\"name\"] = \"device.ip\",\n [\"value\"] = ip_address,\n })\n end\n\n return fields\nend\n\n\n\n\nlocal function getConstantFields(event_type, event) \n if event_type == \"aianalyst/groups\" then\n return get_group_logs_computed_fields(event)\n elseif event_type == \"aianalyst/incidentevents\" then\n return get_incidents_logs_computed_fields(event)\n elseif event_type == \"modelbreaches\" then\n return get_modelbreaches_logs_computed_fields(event)\n elseif event_type == \"status\" then\n return get_status_logs_computed_fields(event)\n end\nend\n\nlocal IGNORE_KEYS = {\n _ob = true,\n _darktrace_event_type = true,\n _darktrace_query_time = true,\n _darktrace_start_time = true,\n _darktrace_end_time = true,\n timestamp = true\n}\n\n\n-- Utility functions\nlocal function split(str, delimiter)\n if not str or str == \"\" then\n return {}\n end\n local result = {}\n for token in string.gmatch(str, \"([^\" .. delimiter .. \"]+)\") do\n table.insert(result, token)\n end\n return result\nend\n\nlocal function getValueByPath(obj, path)\n local current = obj\n for _, part in ipairs(split(path, \".\")) do\n if type(current) ~= \"table\" then\n return nil\n end\n local key = tonumber(part) or part\n current = current[key]\n end\n return current\nend\n\nlocal function setValueByPath(obj, path, value)\n local parts = split(path, \".\")\n local current = obj\n for i = 1, #parts - 1 do\n local key = tonumber(parts[i]) or parts[i]\n if current[key] == nil or type(current[key]) ~= \"table\" then\n current[key] = {}\n end\n current = current[key]\n end\n local last = parts[#parts]\n if value == nil then\n current[last] = nil\n else\n current[last] = value\n end\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n return value\n end\n local copy = {}\n for k, v in pairs(value) do\n if not (ignoreKeys and ignoreKeys[k]) then\n copy[k] = deepCopy(v, ignoreKeys)\n end\n end\n return copy\nend\n\n-- Helper: Encode Lua table to JSON string with field ordering\nlocal function encodeJson(obj, fieldOrder, key)\n if obj == nil then\n return \"null\"\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n if isArray then\n local items = {}\n for i = 1, maxIndex do\n table.insert(items, encodeJson(obj[i], fieldOrder, key))\n end\n return \"[\" .. table.concat(items, \",\") .. \"]\"\n else\n local items = {}\n local fieldOrdering = fieldOrder[key] or {}\n \n -- Phase 1: ordered keys\n for _, fieldName in ipairs(fieldOrdering) do\n local v = obj[fieldName]\n if v ~= nil then\n local encoded = encodeJson(v, fieldOrder, fieldName)\n if encoded ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n end\n end\n end\n \n -- Phase 2: remaining keys (not in fieldOrder)\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrdering) do\n if k == fieldName then\n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local encoded = encodeJson(v, fieldOrder, keyStr)\n if encoded ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n end\n end\n end\n \n return \"{\" .. table.concat(items, \",\") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal GROUPS_FIELD_ORDER = {\n message = {\n \"id\", \"active\", \"acknowledged\", \"pinned\", \"userTriggered\", \"externalTriggered\", \n \"previousIds\", \"incidentEvents\", \"mitreTactics\", \"devices\", \"initialDevices\", \n \"category\", \"groupScore\", \"start\", \"end\", \"edges\"\n },\n incidentEvents = {\n \"uuid\", \"start\", \"title\", \"triggerDid\", \"visible\"\n },\n edges = {\n \"isAction\", \"source\", \"target\", \"start\", \"incidentEvent\", \"description\", \"details\"\n },\n source = {\n \"nodeType\", \"value\"\n },\n target = {\n \"nodeType\", \"value\"\n }\n}\n\nlocal INCIDENT_FIELD_ORDER = {\n message = {\n \"summariser\", \"acknowledged\", \"pinned\", \"createdAt\", \"attackPhases\", \"mitreTactics\",\n \"title\", \"id\", \"children\", \"category\", \"currentGroup\", \"groupCategory\", \"groupScore\",\n \"groupPreviousGroups\", \"activityId\", \"groupingIds\", \"groupByActivity\", \"userTriggered\",\n \"externalTriggered\", \"aiaScore\", \"summary\", \"periods\", \"breachDevices\", \"relatedBreaches\",\n \"details\"\n },\n periods = {\n \"start\", \"end\"\n },\n breachDevices = {\n \"identifier\", \"hostname\", \"ip\", \"mac\", \"subnet\", \"did\", \"sid\"\n },\n relatedBreaches = {\n \"modelName\", \"pbid\", \"threatScore\", \"timestamp\"\n },\n details = {\n \"contents\", \"header\"\n },\n contents = {\n \"key\", \"type\", \"values\"\n },\n values = {\n \"identifier\", \"hostname\", \"ip\", \"mac\", \"subnet\", \"did\", \"sid\"\n }\n}\n\nlocal MODEL_BREACH_FIELD_ORDER = {\n message = {\n \"commentCount\", \"pbid\", \"time\", \"creationTime\", \"model\", \n \"triggeredComponents\", \"score\", \"device\"\n },\n model = {\n \"then\", \"now\"\n },\n -- 'then' is a reserved keyword in Lua, so we use bracket notation\n [\"then\"] = {\n \"name\", \"pid\", \"phid\", \"uuid\", \"logic\", \"throttle\", \"sharedEndpoints\", \n \"actions\", \"tags\", \"interval\", \"delay\", \"sequenced\", \"active\", \"modified\", \n \"activeTimes\", \"autoUpdatable\", \"autoUpdate\", \"autoSuppress\", \"description\", \n \"behaviour\", \"defeats\", \"created\", \"edited\", \"message\", \"version\", \n \"priority\", \"category\", \"compliance\"\n },\n now = {\n \"name\", \"pid\", \"phid\", \"uuid\", \"logic\", \"throttle\", \"sharedEndpoints\", \n \"actions\", \"tags\", \"interval\", \"delay\", \"sequenced\", \"active\", \"modified\", \n \"activeTimes\", \"autoUpdatable\", \"autoUpdate\", \"autoSuppress\", \"description\", \n \"behaviour\", \"defeats\", \"created\", \"edited\", \"message\", \"version\", \n \"priority\", \"category\", \"compliance\"\n },\n actions = {\n \"alert\", \"antigena\", \"breach\", \"model\", \"setPriority\", \"setTag\", \"setType\"\n },\n activeTimes = {\n \"devices\", \"tags\", \"type\", \"version\"\n },\n created = { \"by\" },\n edited = { \"by\" },\n triggeredComponents = {\n \"time\", \"cbid\", \"cid\", \"chid\", \"size\", \"threshold\", \"interval\", \n \"logic\", \"version\", \"metric\", \"triggeredFilters\"\n },\n metric = {\n \"mlid\", \"name\", \"label\"\n },\n triggeredFilters = {\n \"cfid\", \"id\", \"filterType\", \"arguments\", \"comparatorType\", \"trigger\"\n },\n arguments = { \"value\" },\n trigger = { \"value\" },\n device = { \"did\" },\n \n -- Logic structures (Shared by model logic and component logic)\n logic = {\n \"data\", \"type\", \"version\"\n },\n -- Recursive data structure for component logic\n data = {\n \"left\", \"operator\", \"right\"\n },\n left = {\n \"left\", \"operator\", \"right\"\n },\n right = {\n \"left\", \"operator\", \"right\"\n }\n}\n\nlocal STATUS_FIELD_ORDER = {\n message = {\n \"excessTraffic\", \"time\", \"installed\", \"mobileAppConfigured\", \"version\", \"ipAddress\",\n \"modelsUpdated\", \"modelPackageVersion\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\",\n \"hostname\", \"inoculation\", \"applianceOSCode\", \"license\", \"saasConnectorLicense\",\n \"antigenaSaasLicense\", \"syslogTLSSHA1Fingerprint\", \"syslogTLSSHA256Fingerprint\",\n \"antigenaNetworkEnabled\", \"antigenaNetworkLicense\", \"antigenaNetworkRunning\",\n \"logIngestionReplicated\", \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\",\n \"logIngestionTypes\", \"logIngestionMatches\", \"licenseCounts\", \"type\", \"diskUtilization\",\n \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\", \"dataQueue\", \"darkflowQueue\",\n \"networkInterfacesState_eth0\", \"networkInterfacesAddress_eth0\",\n \"networkInterfacesReceived_eth0\", \"networkInterfacesTransmitted_eth0\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\", \"bandwidth2WeekPeakString\",\n \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\", \"processedBandwidthAverage\",\n \"processedBandwidthAverageString\", \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"eventsPerMinuteCurrent\", \"probes\", \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\", \"operatingSystems\",\n \"newDevices4Weeks\", \"newDevices7Days\", \"newDevices24Hours\", \"newDevicesHour\",\n \"activeDevices4Weeks\", \"activeDevices7Days\", \"activeDevices24Hours\", \"activeDevicesHour\",\n \"deviceHostnames\", \"deviceMACAddresses\", \"deviceRecentIPChange\", \"models\", \"modelsBreached\",\n \"modelsSuppressed\", \"devicesModeled\", \"recentUnidirectionalConnections\",\n \"mostRecentCOTPTraffic\", \"mostRecentDCE_RPCTraffic\", \"mostRecentDHCPTraffic\",\n \"mostRecentDNSTraffic\", \"mostRecentFTPTraffic\", \"mostRecentGSSAPITraffic\", \"mostRecentH2Traffic\",\n \"mostRecentHTTPTraffic\", \"mostRecentHTTPSTraffic\", \"mostRecentKERBEROSTraffic\",\n \"mostRecentLDAPTraffic\", \"mostRecentNETLOGONTraffic\", \"mostRecentNTLMTraffic\",\n \"mostRecentNTPTraffic\", \"mostRecentRDPTraffic\", \"mostRecentSMBTraffic\", \"mostRecentSOCKSTraffic\",\n \"mostRecentSSHTraffic\", \"mostRecentSSLTraffic\", \"ignoreAnalysisCredentials\",\n \"internalIPRangeList\", \"internalIPRanges\", \"dnsServers\", \"internalDomains\",\n \"internalAndExternalDomains\", \"proxyServers\", \"subnets\", \"subnetData\"\n },\n probes = {\"1\", \"2\", \"3\"},\n [\"1\"] = {\n \"id\", \"mappedId\", \"antigenaNetworkBlockedConnections\", \"configuredServer\", \"version\",\n \"ipAddress\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\", \"metadata\", \"hostname\",\n \"time\", \"installed\", \"kernel\", \"applianceOSCode\", \"syslogTLSSHA1Fingerprint\",\n \"syslogTLSSHA256Fingerprint\", \"antigenaNetworkRunning\", \"logIngestionReplicated\",\n \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\", \"logIngestionDecryption\",\n \"type\", \"diskUtilization\", \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\",\n \"networkInterfacesState_ens5\", \"networkInterfacesAddress_ens5\",\n \"networkInterfacesReceived_ens5\", \"networkInterfacesTransmitted_ens5\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\",\n \"bandwidth2WeekPeakString\", \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\",\n \"processedBandwidthAverage\", \"processedBandwidthAverageString\",\n \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\",\n \"label\", \"error\", \"lastContact\"\n },\n [\"2\"] = {\n \"id\", \"mappedId\", \"antigenaNetworkBlockedConnections\", \"configuredServer\", \"version\",\n \"ipAddress\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\", \"metadata\", \"hostname\",\n \"time\", \"installed\", \"kernel\", \"applianceOSCode\", \"syslogTLSSHA1Fingerprint\",\n \"syslogTLSSHA256Fingerprint\", \"antigenaNetworkRunning\", \"logIngestionReplicated\",\n \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\", \"logIngestionDecryption\",\n \"type\", \"diskUtilization\", \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\",\n \"networkInterfacesState_ens5\", \"networkInterfacesAddress_ens5\",\n \"networkInterfacesReceived_ens5\", \"networkInterfacesTransmitted_ens5\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\",\n \"bandwidth2WeekPeakString\", \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\",\n \"processedBandwidthAverage\", \"processedBandwidthAverageString\",\n \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\",\n \"label\", \"error\", \"lastContact\"\n },\n [\"3\"] = {\n \"id\", \"mappedId\", \"antigenaNetworkBlockedConnections\", \"configuredServer\", \"version\",\n \"ipAddress\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\", \"metadata\", \"hostname\",\n \"time\", \"installed\", \"kernel\", \"applianceOSCode\", \"syslogTLSSHA1Fingerprint\",\n \"syslogTLSSHA256Fingerprint\", \"antigenaNetworkRunning\", \"logIngestionReplicated\",\n \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\", \"logIngestionDecryption\",\n \"type\", \"diskUtilization\", \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\",\n \"networkInterfacesState_ens5\", \"networkInterfacesAddress_ens5\",\n \"networkInterfacesReceived_ens5\", \"networkInterfacesTransmitted_ens5\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\",\n \"bandwidth2WeekPeakString\", \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\",\n \"processedBandwidthAverage\", \"processedBandwidthAverageString\",\n \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\",\n \"label\", \"error\", \"lastContact\"\n },\n\n antigenaNetworkBlockedConnections = {\n \"attempted\", \"failed\"\n },\n \n subnetData = {\n \"sid\", \"network\", \"devices\", \"clientDevices\", \"mostRecentTraffic\", \"mostRecentDHCP\", \"kerberosQuality\"\n },\n \n eventsPerMinuteCurrent = {\n \"networkConnections\", \"logInputConnections\", \"cSensorConnections\", \"cSensorNotices\",\n \"cSensorDeviceDetails\", \"cSensorModelEvents\", \"networkNotices\", \"networkDeviceDetails\",\n \"networkModelEvents\", \"logInputNotices\", \"logInputDeviceDetails\", \"logInputModelEvents\",\n \"saasNotices\", \"saasModelEvents\"\n },\n \n licenseCounts = {\n \"saas\", \"licenseIPCount\"\n },\n \n saas = {\n \"total\"\n },\n\n -- Dynamic keys; listing those present in the log to ensure order if they appear\n logIngestionTypes = {\n \"TestingDeviceObjects-connectionlogs\", \"TestingDeviceObjects2-connectionlogs\",\n \"TestingDeviceObjects3-connectionlogs\", \"TestingDeviceObjects4-connectionlogs\"\n },\n \n logIngestionMatches = {\n \"TestMatch\", \"TestSrcHostname\", \"TestingDeviceObjects-connectionlogs\"\n }\n}\n\n\nlocal function get_msg_field_ordering(event_type)\n if event_type == \"aianalyst/groups\" then\n return GROUPS_FIELD_ORDER\n elseif event_type == \"aianalyst/incidentevents\" then\n return INCIDENT_FIELD_ORDER\n elseif event_type == \"modelbreaches\" then\n return MODEL_BREACH_FIELD_ORDER\n elseif event_type == \"status\" then\n return STATUS_FIELD_ORDER\n end\nend\n\nlocal function cleanupEmptyNull(obj)\n if type(obj) ~= \"table\" then\n if type(obj) == \"string\" then\n local lower = obj:lower()\n if lower == \"\" or lower == \"null\" then\n return nil\n end\n end\n return obj\n end\n for key, value in pairs(obj) do\n local cleaned = cleanupEmptyNull(value)\n if cleaned == nil then\n obj[key] = nil\n else\n obj[key] = cleaned\n end\n end\n if next(obj) == nil then\n return nil\n end\n return obj\nend\n\nlocal function collectUnmapped(source, target)\n for key, value in pairs(source or {}) do\n if type(value) == \"table\" then\n local nested = {}\n collectUnmapped(value, nested)\n if next(nested) then\n target[key] = nested\n end\n else\n target[key] = value\n end\n end\nend\n\nlocal function getEventType(source)\n if source[\"start\"] then\n return \"aianalyst/groups\"\n elseif source[\"relatedBreaches\"] then\n return \"aianalyst/incidentevents\"\n elseif source[\"time\"] and source[\"triggeredComponents\"] then\n return \"modelbreaches\"\n else\n return \"status\"\n end\nend\n\nfunction processEvent(event)\n if type(event) ~= \"table\" then\n return nil\n end\n\n local MAPPED_FIELDS = {}\n\n local source = deepCopy(event, IGNORE_KEYS)\n\n local event_type = getEventType(source)\n\n local result = {}\n \n for _, mapping in ipairs(COMMON_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n\n if event_type == \"aianalyst/groups\" then\n for _, mapping in ipairs(GROUPS_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n elseif event_type == \"aianalyst/incidentevents\" then\n for _, mapping in ipairs(INCIDENTS_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n elseif event_type == \"modelbreaches\" then\n for _, mapping in ipairs(MODEL_BREACHES_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n elseif event_type == \"status\" then\n for _, mapping in ipairs(STATUS_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n end\n\n for key, value in pairs(getConstantFields(event_type, event)) do\n result[key] = value\n end\n\n -- Remove mapped fields from original event for unmapped collection\n for key, _ in pairs(MAPPED_FIELDS) do\n setValueByPath(event, key, nil)\n end\n \n for key, _ in pairs(IGNORE_KEYS) do\n setValueByPath(event, key, nil)\n end\n\n \n\n local unmapped = {}\n collectUnmapped(event, unmapped)\n unmapped[\"event.type\"] = event_type\n result.unmapped = {}\n if next(unmapped) then\n -- Merge with existing unmapped\n for k, v in pairs(unmapped) do\n result.unmapped[k] = v\n end\n end\n\n result.message = encodeJson(source, get_msg_field_ordering(event_type), \"message\")\n\n if FEATURES.CLEANUP_EMPTY_NULL then\n cleanupEmptyNull(result)\n end\n\n return result\nend", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/darktrace/metadata.yaml b/pipelines/community/transform_ocsf/darktrace/metadata.yaml deleted file mode 100644 index e025630..0000000 --- a/pipelines/community/transform_ocsf/darktrace/metadata.yaml +++ /dev/null @@ -1,51 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Darktrace. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: darktrace - dataSource: Darktrace - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "Syslog" - auth_type: "N/A" - sample_record: "{\n \"time\": 1776656452626,\n \"creationTime\": 1776656333626,\n \"model\": {\n\ - \ \"name\": \"Anomalous File / Internet Facing System File Download\",\n \"description\": \"\ - An internet-facing system has downloaded an unusual file type\",\n \"id\": 861,\n \"version\"\ - : 4,\n \"uuid\": \"789427c6-14d5-4a2b-9e0c-2b0fafb27ed4\"\n },\n \"breachUrl\": \"https://darktrace-07fa459a-0001-01/#modelbreach/71659\"\ - ,\n \"pbid\": 5394290,\n \"score\": 0.722,\n \"device\": {\n \"hostname\": \"PRINTER-971\",\n\ - \ \"ip\": \"10.98.33.252\",\n \"mac\": \"55:de:36:e2:4a:d6\",\n \"type\": \"router\",\n \ - \ \"os\": \"Unknown\"\n },\n \"triggeredComponents\": [\n {\n \"time\": 1776656452626,\n\ - \ \"uid\": \"eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4\",\n \"pid\": 9692,\n \"detail\"\ - : {\n \"fileName\": \"chrome_update.exe\",\n \"fileHash\": \"ea54cb6d1ea944ee9c0a9e70a006c931\"\ - ,\n \"fileSize\": 424534,\n \"downloadSource\": \"http://crypto-miner-4.io/download\"\ - \n }\n },\n {\n \"time\": 1776656452626,\n \"uid\": \"eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4\"\ - ,\n \"pid\": 9692,\n \"detail\": {\n \"connections\": 33,\n \"bytesIn\": 7258485,\n\ - \ \"bytesOut\": 4496939,\n \"duration\": 2974\n }\n }\n ],\n \"commentCount\"\ - : 4,\n \"acknowledged\": true,\n \"category\": \"malware\",\n \"mitreTactics\": [\n \"Execution\"\ - ,\n \"Defense Evasion\"\n ],\n \"tags\": [\n \"darktrace\",\n \"anomaly\",\n \"security\"\ - ,\n \"malware\",\n \"trojan\",\n \"virus\"\n ]\n}" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: darktrace, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/darktrace/sample.json b/pipelines/community/transform_ocsf/darktrace/sample.json deleted file mode 100644 index 89f7fad..0000000 --- a/pipelines/community/transform_ocsf/darktrace/sample.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "time": 1776656452626, - "creationTime": 1776656333626, - "model": { - "name": "Anomalous File / Internet Facing System File Download", - "description": "An internet-facing system has downloaded an unusual file type", - "id": 861, - "version": 4, - "uuid": "789427c6-14d5-4a2b-9e0c-2b0fafb27ed4" - }, - "breachUrl": "https://darktrace-07fa459a-0001-01/#modelbreach/71659", - "pbid": 5394290, - "score": 0.722, - "device": { - "hostname": "PRINTER-971", - "ip": "10.98.33.252", - "mac": "55:de:36:e2:4a:d6", - "type": "router", - "os": "Unknown" - }, - "triggeredComponents": [ - { - "time": 1776656452626, - "uid": "eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4", - "pid": 9692, - "detail": { - "fileName": "chrome_update.exe", - "fileHash": "ea54cb6d1ea944ee9c0a9e70a006c931", - "fileSize": 424534, - "downloadSource": "http://crypto-miner-4.io/download" - } - }, - { - "time": 1776656452626, - "uid": "eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4", - "pid": 9692, - "detail": { - "connections": 33, - "bytesIn": 7258485, - "bytesOut": 4496939, - "duration": 2974 - } - } - ], - "commentCount": 4, - "acknowledged": true, - "category": "malware", - "mitreTactics": [ - "Execution", - "Defense Evasion" - ], - "tags": [ - "darktrace", - "anomaly", - "security", - "malware", - "trojan", - "virus" - ] -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/darktrace/serializer.lua b/pipelines/community/transform_ocsf/darktrace/serializer.lua deleted file mode 100644 index d85962b..0000000 --- a/pipelines/community/transform_ocsf/darktrace/serializer.lua +++ /dev/null @@ -1,987 +0,0 @@ --- Darktrace 1.1.0 OCSF Serializer - -local FEATURES = { - CLEANUP_EMPTY_NULL = true, -} - -local COMMON_MAPPING = { - {source = "action", target = "action"}, - {source = "action_id", target = "action_id"}, - {source = "category_uid", target = "category_uid"}, - {source = "category_name", target = "category_name"}, - {source = "class_uid", target = "class_uid"}, - {source = "class_name", target = "class_name"}, - {source = "activity_id", target = "activity_id"}, - {source = "activity_name", target = "activity_name"}, - {source = "type_uid", target = "type_uid"}, - {source = "type_name", target = "type_name"}, - {source = "severity_id", target = "severity_id"}, - {source = "metadata.product.vendor_name", target = "metadata.product.vendor_name"}, - {source = "metadata.product.name", target = "metadata.product.name"}, - {source = "metadata.version", target = "metadata.version"}, - {source = "metadata.original_time", target = "metadata.original_time"}, - {source = "dataSource.category", target = "dataSource.category"}, - {source = "dataSource.name", target = "dataSource.name"}, - {source = "dataSource.vendor", target = "dataSource.vendor"}, - {source = "site.id", target = "site.id"}, - {source = "message", target = "message"}, - {source = "user.type_id", target = "user.type_id"}, - {source = "observables", target = "observables"} -} - -local GROUPS_LOGS_MAPPING = { - {source = "id", target = "metadata.uid"}, - {source = "mitreTactics", target = "attacks.tactics"}, - {source = "category", target = "severity"}, - {source = "start", target = "start_time"}, - {source = "start_time", target = "time"}, - {source = "end", target = "end_time"}, - {source = "finding_info.uid", target = "finding_info.uid"}, - {source = "finding_info.first_seen_time", target = "finding_info.first_seen_time"}, - {source = "finding_info.title", target = "finding_info.title"}, - {source = "finding_info.product_uid", target = "finding_info.product_uid"}, - {source = "finding_info.desc", target = "finding_info.desc"} -} - -local INCIDENTS_LOGS_MAPPING = { - {source = "createdAt", target = "finding_info.created_time"}, - {source = "mitreTactics", target = "attacks.tactics"}, - {source = "start_time", target = "start_time"}, - {source = "end_time", target = "end_time"}, - {source = "device.namespace_pid", target = "device.namespace_pid"}, - {source = "device.hostname", target = "device.hostname"}, - {source = "device.ip", target = "device.ip"}, - {source = "device.mac", target = "device.mac"}, - {source = "device.subnet", target = "device.subnet"}, - {source = "device.uid", target = "device.uid"}, - {source = "device.subnet_uid", target = "device.subnet_uid"}, - {source = "device.type_id", target = "device.type_id"}, - {source = "authorizations", target = "authorizations"}, - {source = "risk_score", target = "risk_score"}, - {source = "id", target = "finding_info.uid"}, - {source = "title", target = "finding_info.title"}, - {source = "finding_info.first_seen_time", target = "finding_info.first_seen_time"}, - {source = "finding_info.product_uid", target = "finding_info.product_uid"}, - {source = "summary", target = "finding_info.desc"}, - {source = "severity", target = "severity"}, - {source = "metadata.original_time", target = "time"} -} - -local MODEL_BREACHES_LOGS_MAPPING = { - {source = "authorizations", target = "authorizations"}, - {source = "time", target = "metadata.original_time"}, - {source = "creationTime", target = "finding_info.created_time"}, - {source = "resources", target = "resources"}, - {source = "resources_result", target = "resources_result"}, - {source = "severity", target = "severity"}, - {source = "device.uid", target = "device.uid"}, - {source = "device.type_id", target = "device.type_id"} -} - -local STATUS_LOGS_MAPPING = { - {source = "ipAddress", target = "device.ip"}, - {source = "hostname", target = "device.hostname"}, - {source = "uuid", target = "device.uid"}, - {source = "device.type_id", target = "device.type_id"}, - {source = "device.type", target = "device.type"}, - {source = "device.subnet_uid", target = "device.subnet_uid"}, - {source = "device.network_interfaces", target = "device.network_interfaces"}, - {source = "severity", target = "severity"} -} - -local function convert_to_epoch_milliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+) (%d+):(%d+)") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - --- Constant Fields - -local function get_group_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["uid"] = event["id"], - ["original_time"] = type(event["start"]) == "number" and event["start"] or convert_to_epoch_milliseconds(event["start"] or ""), - }, - ["action"] = "Other", - ["action_id"] = 99, - ["activity_id"] = 1, - ["activity_name"] = "Create", - ["event.type"] = "Create", - ["category_name"] = "Findings", - ["category_uid"] = 2, - ["class_name"] = "Detection Finding", - ["class_uid"] = 2004, - ["type_name"] = "Detection Finding: Create", - ["type_uid"] = 200401, - ["severity_id"] = 99, - ["time"] = type(event["start"]) == "number" and event["start"] or convert_to_epoch_milliseconds(event["start"] or "") - } - - local incident_events = event["incidentEvents"] or {} - -- Check if list is not empty (Lua tables are 1-indexed) - if #incident_events > 0 then - local incident = incident_events[1] - fields["finding_info"] = { - ["uid"] = incident["uuid"], - ["first_seen_time"] = incident["start"], - ["title"] = incident["title"], - ["product_uid"] = incident["triggerDid"], - } - end - - local observables_array = {} - for _, incident in ipairs(incident_events) do - -- Resource UID observable object - local uuid_val = incident["uuid"] or "" - if uuid_val ~= "" then - table.insert(observables_array, {type_id = 10, type = "Resource UID", name = "finding_info.uid", value = uuid_val}) - end - - -- Process observable object - local title_val = incident["title"] or "" - if title_val ~= "" then - table.insert(observables_array, {type_id = 25, type = "Process", name = "finding_info.title", value = title_val}) - end - end - - if #observables_array > 0 then - fields["observables"] = observables_array - end - - return fields -end - -local function get_incidents_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["original_time"] = type(event["relatedBreaches"][1]["timestamp"]) == "number" and event["relatedBreaches"][1]["timestamp"] or convert_to_epoch_milliseconds(event["relatedBreaches"][1]["timestamp"] or "") - }, - ["action"] = "Other", - ["action_id"] = 99, - ["activity_id"] = 1, - ["activity_name"] = "Create", - ["event.type"] = "Create", - ["category_name"] = "Findings", - ["category_uid"] = 2, - ["class_name"] = "Detection Finding", - ["class_uid"] = 2004, - ["type_name"] = "Detection Finding: Create", - ["type_uid"] = 200401, - ["severity_id"] = 99, - ["time"] = type(event["relatedBreaches"][1]["timestamp"]) == "number" and event["relatedBreaches"][1]["timestamp"] or convert_to_epoch_milliseconds(event["relatedBreaches"][1]["timestamp"] or "") - } - - local periods = event["periods"] or {} - if #periods > 0 then - fields["start_time"] = periods[1]["start"] - fields["end_time"] = periods[1]["end"] - end - - local related_breaches = event["relatedBreaches"] or {} - if #related_breaches > 0 then - fields["authorizations"] = { - { - ["policy"] = { - ["name"] = related_breaches[1]["modelName"], - ["uid"] = related_breaches[1]["pbid"] - } - } - } - fields["risk_score"] = related_breaches[1]["threatScore"] - end - - local breach_devices = event["breachDevices"] or {} - if #breach_devices > 0 then - fields["device"] = { - ["namespace_pid"] = breach_devices[1]["identifier"], - ["hostname"] = breach_devices[1]["hostname"], - ["ip"] = breach_devices[1]["ip"], - ["mac"] = breach_devices[1]["mac"], - ["subnet"] = breach_devices[1]["subnet"], - ["uid"] = breach_devices[1]["did"], - ["subnet_uid"] = breach_devices[1]["sid"], - ["type_id"] = 0 - } - end - - local observables_array = {} - for _, incident in ipairs(breach_devices) do - -- Hostname observable object - local hostname_val = incident["hostname"] or "" - if hostname_val ~= "" then - local hostname_obj = {} - hostname_obj["type_id"] = 1 - hostname_obj["type"] = "Hostname" - hostname_obj["name"] = "device.hostname" - hostname_obj["value"] = hostname_val - table.insert(observables_array, hostname_obj) - end - - -- IP Address observable object - local ip_val = incident["ip"] or "" - if ip_val ~= "" then - local ip_obj = {} - ip_obj["type_id"] = 2 - ip_obj["type"] = "IP Address" - ip_obj["name"] = "device.ip" - ip_obj["value"] = ip_val - table.insert(observables_array, ip_obj) - end - - -- MAC Address observable object - local mac_val = incident["mac"] or "" - if mac_val ~= "" then - local mac_obj = {} - mac_obj["type_id"] = 3 - mac_obj["type"] = "MAC Address" - mac_obj["name"] = "device.mac" - mac_obj["value"] = mac_val - table.insert(observables_array, mac_obj) - end - end - - if #observables_array > 0 then - fields["observables"] = observables_array - end - - return fields -end - -local function get_modelbreaches_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["original_time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or "") - }, - ["action"] = "Other", - ["action_id"] = 99, - ["activity_id"] = 1, - ["activity_name"] = "Create", - ["event.type"] = "Create", - ["category_name"] = "Findings", - ["category_uid"] = 2, - ["class_name"] = "Detection Finding", - ["class_uid"] = 2004, - ["type_name"] = "Detection Finding: Create", - ["type_uid"] = 200401, - ["severity_id"] = 99, - ["time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or "") - } - - fields["authorizations"] = { - { - ["policy"] = { - ["uid"] = event["pbid"] - } - } - } - - local model = event["model"] or {} - local model_then = model["then"] - -- In Lua, empty tables are true, so check if not nil. - -- If 'then' might be an empty table {}, you might need next(model_then) check, - -- but usually API responses omit keys or return nil if missing. - if model_then then - fields["resources"] = { - { - ["name"] = model_then["name"], - ["uid"] = model_then["uuid"], - ["version"] = model_then["version"], - ["data"] = model_then - } - } - end - - local model_now = model["now"] - -- Note: Original Python code had a bug: 'if model_then:' check for 'resources_result' - -- I assumed you wanted 'if model_now:' here based on context. - -- If you want strict translation of the bug, change 'model_now' to 'model_then' in the if condition. - if model_now then - fields["resources_result"] = { - { - ["name"] = model_now["name"], - ["uid"] = model_now["uuid"], - ["version"] = model_now["version"], - ["data"] = model_now - } - } - end - - local device = event["device"] or {} - local device_did = device["did"] - if device_did then - fields["device"] = { - ["uid"] = device_did, - ["type_id"] = 0 - } - end - - return fields -end - -local function get_status_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["original_time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or "") - }, - ["device"] = { - ["type_id"] = 99, - ["type"] = event["type"], - ["hostname"] = event["hostname"], - ["ip"] = event["ipAddress"], - ["network_interfaces"] = { - { - ["ip"] = event["networkInterfacesAddress_eth0"], - ["type_id"] = 1, - ["type"] = "Wired" - } - } - }, - ["time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or ""), - ["activity_id"] = 1, - ["activity_name"] = "Log", - ["event.type"] = "Log", - ["category_name"] = "Discovery", - ["category_uid"] = 5, - ["class_name"] = "Device Config State", - ["class_uid"] = 5002, - ["type_name"] = "Device Config State: Log", - ["type_uid"] = 500201, - ["severity_id"] = 99, - } - - -- Initialize device table - - local subnet_data = event["subnetData"] or {} - if #subnet_data >= 1 then - local sid = subnet_data[1]["sid"] - fields["device"]["subnet_uid"] = sid - end - - fields["observables"] = {} - - local hostname = event["hostname"] - if hostname and hostname ~= "" then - table.insert(fields["observables"], { - ["type_id"] = 1, - ["type"] = "Hostname", - ["name"] = "device.hostname", - ["value"] = hostname, - }) - end - - local ip_address = event["ipAddress"] - if ip_address and ip_address ~= "" then - table.insert(fields["observables"], { - ["type_id"] = 2, - ["type"] = "IP Address", - ["name"] = "device.ip", - ["value"] = ip_address, - }) - end - - return fields -end - - - - -local function getConstantFields(event_type, event) - if event_type == "aianalyst/groups" then - return get_group_logs_computed_fields(event) - elseif event_type == "aianalyst/incidentevents" then - return get_incidents_logs_computed_fields(event) - elseif event_type == "modelbreaches" then - return get_modelbreaches_logs_computed_fields(event) - elseif event_type == "status" then - return get_status_logs_computed_fields(event) - end -end - -local IGNORE_KEYS = { - _ob = true, - _darktrace_event_type = true, - _darktrace_query_time = true, - _darktrace_start_time = true, - _darktrace_end_time = true, - timestamp = true -} - - --- Utility functions -local function split(str, delimiter) - if not str or str == "" then - return {} - end - local result = {} - for token in string.gmatch(str, "([^" .. delimiter .. "]+)") do - table.insert(result, token) - end - return result -end - -local function getValueByPath(obj, path) - local current = obj - for _, part in ipairs(split(path, ".")) do - if type(current) ~= "table" then - return nil - end - local key = tonumber(part) or part - current = current[key] - end - return current -end - -local function setValueByPath(obj, path, value) - local parts = split(path, ".") - local current = obj - for i = 1, #parts - 1 do - local key = tonumber(parts[i]) or parts[i] - if current[key] == nil or type(current[key]) ~= "table" then - current[key] = {} - end - current = current[key] - end - local last = parts[#parts] - if value == nil then - current[last] = nil - else - current[last] = value - end -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - --- Helper: Encode Lua table to JSON string with field ordering -local function encodeJson(obj, fieldOrder, key) - if obj == nil then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray then - local items = {} - for i = 1, maxIndex do - table.insert(items, encodeJson(obj[i], fieldOrder, key)) - end - return "[" .. table.concat(items, ",") .. "]" - else - local items = {} - local fieldOrdering = fieldOrder[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrdering) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldOrder, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - -- Phase 2: remaining keys (not in fieldOrder) - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrdering) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, fieldOrder, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - return "{" .. table.concat(items, ",") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local GROUPS_FIELD_ORDER = { - message = { - "id", "active", "acknowledged", "pinned", "userTriggered", "externalTriggered", - "previousIds", "incidentEvents", "mitreTactics", "devices", "initialDevices", - "category", "groupScore", "start", "end", "edges" - }, - incidentEvents = { - "uuid", "start", "title", "triggerDid", "visible" - }, - edges = { - "isAction", "source", "target", "start", "incidentEvent", "description", "details" - }, - source = { - "nodeType", "value" - }, - target = { - "nodeType", "value" - } -} - -local INCIDENT_FIELD_ORDER = { - message = { - "summariser", "acknowledged", "pinned", "createdAt", "attackPhases", "mitreTactics", - "title", "id", "children", "category", "currentGroup", "groupCategory", "groupScore", - "groupPreviousGroups", "activityId", "groupingIds", "groupByActivity", "userTriggered", - "externalTriggered", "aiaScore", "summary", "periods", "breachDevices", "relatedBreaches", - "details" - }, - periods = { - "start", "end" - }, - breachDevices = { - "identifier", "hostname", "ip", "mac", "subnet", "did", "sid" - }, - relatedBreaches = { - "modelName", "pbid", "threatScore", "timestamp" - }, - details = { - "contents", "header" - }, - contents = { - "key", "type", "values" - }, - values = { - "identifier", "hostname", "ip", "mac", "subnet", "did", "sid" - } -} - -local MODEL_BREACH_FIELD_ORDER = { - message = { - "commentCount", "pbid", "time", "creationTime", "model", - "triggeredComponents", "score", "device" - }, - model = { - "then", "now" - }, - -- 'then' is a reserved keyword in Lua, so we use bracket notation - ["then"] = { - "name", "pid", "phid", "uuid", "logic", "throttle", "sharedEndpoints", - "actions", "tags", "interval", "delay", "sequenced", "active", "modified", - "activeTimes", "autoUpdatable", "autoUpdate", "autoSuppress", "description", - "behaviour", "defeats", "created", "edited", "message", "version", - "priority", "category", "compliance" - }, - now = { - "name", "pid", "phid", "uuid", "logic", "throttle", "sharedEndpoints", - "actions", "tags", "interval", "delay", "sequenced", "active", "modified", - "activeTimes", "autoUpdatable", "autoUpdate", "autoSuppress", "description", - "behaviour", "defeats", "created", "edited", "message", "version", - "priority", "category", "compliance" - }, - actions = { - "alert", "antigena", "breach", "model", "setPriority", "setTag", "setType" - }, - activeTimes = { - "devices", "tags", "type", "version" - }, - created = { "by" }, - edited = { "by" }, - triggeredComponents = { - "time", "cbid", "cid", "chid", "size", "threshold", "interval", - "logic", "version", "metric", "triggeredFilters" - }, - metric = { - "mlid", "name", "label" - }, - triggeredFilters = { - "cfid", "id", "filterType", "arguments", "comparatorType", "trigger" - }, - arguments = { "value" }, - trigger = { "value" }, - device = { "did" }, - - -- Logic structures (Shared by model logic and component logic) - logic = { - "data", "type", "version" - }, - -- Recursive data structure for component logic - data = { - "left", "operator", "right" - }, - left = { - "left", "operator", "right" - }, - right = { - "left", "operator", "right" - } -} - -local STATUS_FIELD_ORDER = { - message = { - "excessTraffic", "time", "installed", "mobileAppConfigured", "version", "ipAddress", - "modelsUpdated", "modelPackageVersion", "bundleVersion", "bundleDate", "bundleInstalledDate", - "hostname", "inoculation", "applianceOSCode", "license", "saasConnectorLicense", - "antigenaSaasLicense", "syslogTLSSHA1Fingerprint", "syslogTLSSHA256Fingerprint", - "antigenaNetworkEnabled", "antigenaNetworkLicense", "antigenaNetworkRunning", - "logIngestionReplicated", "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", - "logIngestionTypes", "logIngestionMatches", "licenseCounts", "type", "diskUtilization", - "uptime", "systemUptime", "load", "cpu", "memoryUsed", "dataQueue", "darkflowQueue", - "networkInterfacesState_eth0", "networkInterfacesAddress_eth0", - "networkInterfacesReceived_eth0", "networkInterfacesTransmitted_eth0", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", "bandwidth2WeekPeakString", - "processedBandwidthCurrent", "processedBandwidthCurrentString", "processedBandwidthAverage", - "processedBandwidthAverageString", "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "eventsPerMinuteCurrent", "probes", "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", "operatingSystems", - "newDevices4Weeks", "newDevices7Days", "newDevices24Hours", "newDevicesHour", - "activeDevices4Weeks", "activeDevices7Days", "activeDevices24Hours", "activeDevicesHour", - "deviceHostnames", "deviceMACAddresses", "deviceRecentIPChange", "models", "modelsBreached", - "modelsSuppressed", "devicesModeled", "recentUnidirectionalConnections", - "mostRecentCOTPTraffic", "mostRecentDCE_RPCTraffic", "mostRecentDHCPTraffic", - "mostRecentDNSTraffic", "mostRecentFTPTraffic", "mostRecentGSSAPITraffic", "mostRecentH2Traffic", - "mostRecentHTTPTraffic", "mostRecentHTTPSTraffic", "mostRecentKERBEROSTraffic", - "mostRecentLDAPTraffic", "mostRecentNETLOGONTraffic", "mostRecentNTLMTraffic", - "mostRecentNTPTraffic", "mostRecentRDPTraffic", "mostRecentSMBTraffic", "mostRecentSOCKSTraffic", - "mostRecentSSHTraffic", "mostRecentSSLTraffic", "ignoreAnalysisCredentials", - "internalIPRangeList", "internalIPRanges", "dnsServers", "internalDomains", - "internalAndExternalDomains", "proxyServers", "subnets", "subnetData" - }, - probes = {"1", "2", "3"}, - ["1"] = { - "id", "mappedId", "antigenaNetworkBlockedConnections", "configuredServer", "version", - "ipAddress", "bundleVersion", "bundleDate", "bundleInstalledDate", "metadata", "hostname", - "time", "installed", "kernel", "applianceOSCode", "syslogTLSSHA1Fingerprint", - "syslogTLSSHA256Fingerprint", "antigenaNetworkRunning", "logIngestionReplicated", - "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", "logIngestionDecryption", - "type", "diskUtilization", "uptime", "systemUptime", "load", "cpu", "memoryUsed", - "networkInterfacesState_ens5", "networkInterfacesAddress_ens5", - "networkInterfacesReceived_ens5", "networkInterfacesTransmitted_ens5", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", - "bandwidth2WeekPeakString", "processedBandwidthCurrent", "processedBandwidthCurrentString", - "processedBandwidthAverage", "processedBandwidthAverageString", - "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", - "label", "error", "lastContact" - }, - ["2"] = { - "id", "mappedId", "antigenaNetworkBlockedConnections", "configuredServer", "version", - "ipAddress", "bundleVersion", "bundleDate", "bundleInstalledDate", "metadata", "hostname", - "time", "installed", "kernel", "applianceOSCode", "syslogTLSSHA1Fingerprint", - "syslogTLSSHA256Fingerprint", "antigenaNetworkRunning", "logIngestionReplicated", - "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", "logIngestionDecryption", - "type", "diskUtilization", "uptime", "systemUptime", "load", "cpu", "memoryUsed", - "networkInterfacesState_ens5", "networkInterfacesAddress_ens5", - "networkInterfacesReceived_ens5", "networkInterfacesTransmitted_ens5", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", - "bandwidth2WeekPeakString", "processedBandwidthCurrent", "processedBandwidthCurrentString", - "processedBandwidthAverage", "processedBandwidthAverageString", - "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", - "label", "error", "lastContact" - }, - ["3"] = { - "id", "mappedId", "antigenaNetworkBlockedConnections", "configuredServer", "version", - "ipAddress", "bundleVersion", "bundleDate", "bundleInstalledDate", "metadata", "hostname", - "time", "installed", "kernel", "applianceOSCode", "syslogTLSSHA1Fingerprint", - "syslogTLSSHA256Fingerprint", "antigenaNetworkRunning", "logIngestionReplicated", - "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", "logIngestionDecryption", - "type", "diskUtilization", "uptime", "systemUptime", "load", "cpu", "memoryUsed", - "networkInterfacesState_ens5", "networkInterfacesAddress_ens5", - "networkInterfacesReceived_ens5", "networkInterfacesTransmitted_ens5", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", - "bandwidth2WeekPeakString", "processedBandwidthCurrent", "processedBandwidthCurrentString", - "processedBandwidthAverage", "processedBandwidthAverageString", - "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", - "label", "error", "lastContact" - }, - - antigenaNetworkBlockedConnections = { - "attempted", "failed" - }, - - subnetData = { - "sid", "network", "devices", "clientDevices", "mostRecentTraffic", "mostRecentDHCP", "kerberosQuality" - }, - - eventsPerMinuteCurrent = { - "networkConnections", "logInputConnections", "cSensorConnections", "cSensorNotices", - "cSensorDeviceDetails", "cSensorModelEvents", "networkNotices", "networkDeviceDetails", - "networkModelEvents", "logInputNotices", "logInputDeviceDetails", "logInputModelEvents", - "saasNotices", "saasModelEvents" - }, - - licenseCounts = { - "saas", "licenseIPCount" - }, - - saas = { - "total" - }, - - -- Dynamic keys; listing those present in the log to ensure order if they appear - logIngestionTypes = { - "TestingDeviceObjects-connectionlogs", "TestingDeviceObjects2-connectionlogs", - "TestingDeviceObjects3-connectionlogs", "TestingDeviceObjects4-connectionlogs" - }, - - logIngestionMatches = { - "TestMatch", "TestSrcHostname", "TestingDeviceObjects-connectionlogs" - } -} - - -local function get_msg_field_ordering(event_type) - if event_type == "aianalyst/groups" then - return GROUPS_FIELD_ORDER - elseif event_type == "aianalyst/incidentevents" then - return INCIDENT_FIELD_ORDER - elseif event_type == "modelbreaches" then - return MODEL_BREACH_FIELD_ORDER - elseif event_type == "status" then - return STATUS_FIELD_ORDER - end -end - -local function cleanupEmptyNull(obj) - if type(obj) ~= "table" then - if type(obj) == "string" then - local lower = obj:lower() - if lower == "" or lower == "null" then - return nil - end - end - return obj - end - for key, value in pairs(obj) do - local cleaned = cleanupEmptyNull(value) - if cleaned == nil then - obj[key] = nil - else - obj[key] = cleaned - end - end - if next(obj) == nil then - return nil - end - return obj -end - -local function collectUnmapped(source, target) - for key, value in pairs(source or {}) do - if type(value) == "table" then - local nested = {} - collectUnmapped(value, nested) - if next(nested) then - target[key] = nested - end - else - target[key] = value - end - end -end - -local function getEventType(source) - if source["start"] then - return "aianalyst/groups" - elseif source["relatedBreaches"] then - return "aianalyst/incidentevents" - elseif source["time"] and source["triggeredComponents"] then - return "modelbreaches" - else - return "status" - end -end - -function processEvent(event) - if type(event) ~= "table" then - return nil - end - - local MAPPED_FIELDS = {} - - local source = deepCopy(event, IGNORE_KEYS) - - local event_type = getEventType(source) - - local result = {} - - for _, mapping in ipairs(COMMON_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - - if event_type == "aianalyst/groups" then - for _, mapping in ipairs(GROUPS_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - elseif event_type == "aianalyst/incidentevents" then - for _, mapping in ipairs(INCIDENTS_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - elseif event_type == "modelbreaches" then - for _, mapping in ipairs(MODEL_BREACHES_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - elseif event_type == "status" then - for _, mapping in ipairs(STATUS_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - end - - for key, value in pairs(getConstantFields(event_type, event)) do - result[key] = value - end - - -- Remove mapped fields from original event for unmapped collection - for key, _ in pairs(MAPPED_FIELDS) do - setValueByPath(event, key, nil) - end - - for key, _ in pairs(IGNORE_KEYS) do - setValueByPath(event, key, nil) - end - - - - local unmapped = {} - collectUnmapped(event, unmapped) - unmapped["event.type"] = event_type - result.unmapped = {} - if next(unmapped) then - -- Merge with existing unmapped - for k, v in pairs(unmapped) do - result.unmapped[k] = v - end - end - - result.message = encodeJson(source, get_msg_field_ordering(event_type), "message") - - if FEATURES.CLEANUP_EMPTY_NULL then - cleanupEmptyNull(result) - end - - return result -end \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/gcp_audit_logs.json b/pipelines/community/transform_ocsf/gcp_audit_logs/gcp_audit_logs.json deleted file mode 100644 index 5749ebc..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/gcp_audit_logs.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Gcp Audit Logs", - "grade": { - "letter": "F", - "score": 25, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Gcp Audit Logs. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "gcp", - "source_name": "gcp_audit_logs", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "gcp-audit-logs-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- GCP Audit Log OCSF 1.0.0 Schema Serializer\nlocal FEATURES = {\n FLATTEN_EVENT_TYPE = true,\n CLEANUP_EMPTY_NULL = true,\n}\n\nlocal BaseEvent = {\n CATEGORY_UID = 3,\n CATEGORY_NAME = \"Identity & Access Management\",\n TIMEZONE_OFFSET = 0,\n STATUS_ID = 0,\n STATUS = \"Unknown\"\n}\n\nlocal DataSourceEvent = {\n VENDOR = \"GCP\",\n NAME = \"GCP Audit\",\n CATEGORY = \"security\"\n}\n\nlocal MetaDataEvent = {\n LOG_PROVIDER = \"GCP Audit\",\n PRODUCT_LANG = \"en\",\n PRODUCT_NAME = \"GCP Audit\",\n PRODUCT_VENDOR_NAME = \"GCP\",\n VERSION = \"1.0.0\"\n}\n\nlocal Mapping = {\n SEVERITY = {\n [\"INFO\"] = {1, \"Informational\"},\n [\"CRITICAL\"] = {5, \"Critical\"}\n },\n ACTIVITY = {\n admin_activity = {99, \"AdminActivity\"},\n data_access = {99, \"DataAccess\"},\n system_event = {99, \"SystemEvent\"},\n policy_denied = {99, \"PolicyDenied\"}\n },\n TYPE = {\n admin_activity = {300499, \"Entity Management: AdminActivity\"},\n data_access = {300499, \"Entity Management: DataAccess\"},\n system_event = {300499, \"Entity Management: SystemEvent\"},\n policy_denied = {300599, \"User Access Management: PolicyDenied\"}\n },\n CLASS = {\n admin_activity = {3004, \"Entity Management\"},\n data_access = {3004, \"Entity Management\"},\n system_event = {3004, \"Entity Management\"},\n policy_denied = {3005, \"User Access Management\"}\n }\n}\n\nlocal MAPPED_FIELDS = {}\n\nlocal FIELD_ORDERS = {\n root = {\n \"activity_id\",\"activity_name\",\"category_uid\",\"category_name\",\"class_uid\",\"class_name\",\"severity_id\",\"type_uid\",\"type_name\",\"dataSource\",\"site.id\",\"metadata.product.name\",\"metadata.product.vendor_name\",\"metadata.version\",\"actor.user.email_addr\",\"device.ip\",\"api.service.name\",\"entity.name\",\"resource.name\",\"metadata.uid\",\"metadata.original_time\",\"metadata.logged_time\",\"metadata.log_name\",\"metadata.event_code\",\"api.operation\",\"cloud.provider\",\"event.type\",\"message\",\"status_code\",\"status\",\"status_detail\",\"time\"\n },\n metadata = {\"original_time\", \"logged_time\", \"log_name\", \"event_code\", \"uid\", \"product\", \"version\", \"currentLocations\", \"deviceState\", \"@type\", \"violationReason\", \"intermediateServices\", \"securityPolicyInfo\", \"resourceNames\", \"ingressViolations\", \"vpcServiceControlsUniqueId\"},\n product = {\"name\", \"vendor_name\"},\n dataSource = {\"category\", \"name\", \"vendor\"},\n site = {\"id\"},\n event = {\"type\"},\n message = {\"protoPayload\", \"insertId\", \"resource\", \"timestamp\", \"severity\", \"logName\", \"operation\", \"receiveTimestamp\"},\n protoPayload = {\"@type\", \"status\", \"authenticationInfo\", \"requestMetadata\", \"serviceName\", \"methodName\", \"authorizationInfo\", \"resourceName\", \"resourceLocation\", \"request\", \"metadata\"},\n status = {\"code\", \"message\", \"details\"},\n authenticationInfo = {\"principalEmail\"},\n requestMetadata = {\"callerIp\", \"callerSuppliedUserAgent\", \"requestAttributes\", \"destinationAttributes\"},\n requestAttributes = {\"time\", \"reason\", \"auth\"},\n authorizationInfo = {\"resource\", \"permission\", \"granted\", \"resourceAttributes\"},\n resource = {\"type\", \"labels\"},\n operation = {\"id\", \"producer\", \"first\"},\n labels = {\"method\", \"service\", \"project_id\", \"location\", \"bucket_name\", \"subnetwork_name\", \"subnetwork_id\"},\n request = {\"@type\"},\n preconditionFailure = {\"@type\", \"violations\"},\n details = {\"@type\", \"violations\"},\n violations = {\"description\", \"type\"},\n ingressViolations = {\"targetResource\", \"servicePerimeter\"},\n securityPolicyInfo = {\"organizationId\", \"servicePerimeterName\"},\n}\n\n\nlocal FIELD_MAPPINGS = {\n {source = \"activity_id\", target = \"activity_id\"},\n {source = \"activity_name\", target = \"activity_name\"},\n {source = \"category_uid\", target = \"category_uid\"},\n {source = \"category_name\", target = \"category_name\"},\n {source = \"class_uid\", target = \"class_uid\"},\n {source = \"class_name\", target = \"class_name\"},\n {source = \"severity_id\", target = \"severity_id\"},\n {source = \"type_uid\", target = \"type_uid\"},\n {source = \"status_code\", target = \"status_code\"},\n {source = \"status\", target = \"status\"},\n {source = \"status_detail\", target = \"status_detail\"},\n {source = \"dataSource\", target = \"dataSource\"},\n {source = \"site.id\", target = \"site.id\"},\n {source = \"metadata.product.name\", target = \"metadata.product.name\"},\n {source = \"metadata.product.vendor_name\", target = \"metadata.product.vendor_name\"},\n {source = \"metadata.version\", target = \"metadata.version\"},\n {source = \"metadata.uid\", target = \"metadata.uid\"},\n {source = \"metadata.original_time\", target = \"metadata.original_time\"},\n {source = \"metadata.logged_time\", target = \"metadata.logged_time\"},\n {source = \"metadata.log_name\", target = \"metadata.log_name\"},\n {source = \"metadata.event_code\", target = \"metadata.event_code\"},\n {source = \"actor.user.email_addr\", target = \"actor.user.email_addr\"},\n {source = \"device.ip\", target = \"device.ip\"},\n {source = \"api.service.name\", target = \"api.service.name\"},\n {source = \"api.operation\", target = \"api.operation\"},\n {source = \"entity.name\", target = \"entity.name\"},\n {source = \"resource.name\", target = \"resource.name\"},\n {source = \"cloud.provider\", target = \"cloud.provider\"},\n {source = \"time\", target = \"time\"},\n {source = \"message\", target = \"message\"}\n --{source = \"event.type\", target = \"event.type\"},\n\n}\n\nlocal IGNORE_KEYS = {\n _ob = true,\n message_id = true,\n}\n\nlocal function split(str, delimiter)\n if not str or str == \"\" then\n return {}\n end\n local result = {}\n for token in string.gmatch(str, \"([^\" .. delimiter .. \"]+)\") do\n table.insert(result, token)\n end\n return result\nend\n\nlocal function getValueByPath(obj, path)\n local current = obj\n for _, part in ipairs(split(path, \".\")) do\n\n if type(current) ~= \"table\" then\n return nil\n end\n\n local key = tonumber(part) or part\n\n current = current[key]\n end\n return current\nend\n\nlocal function setValueByPath(obj, path, value)\n local parts = split(path, \".\")\n local current = obj\n for i = 1, #parts - 1 do\n local key = tonumber(parts[i]) or parts[i]\n if current[key] == nil or type(current[key]) ~= \"table\" then\n current[key] = {}\n end\n current = current[key]\n end\n local last = parts[#parts]\n if value == nil then\n current[last] = nil\n else\n current[last] = value\n end\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n return value\n end\n\n local copy = {}\n for k, v in pairs(value) do\n if not (ignoreKeys and ignoreKeys[k]) then\n copy[k] = deepCopy(v, ignoreKeys)\n end\n end\n return copy\nend\n\nlocal function convertUtcToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\nlocal function encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" or obj == \"\" then\n return nil\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n\n if isArray then\n local items = {}\n for i = 1, maxIndex do\n local elementKey = key or tostring(i)\n local encoded = encodeJson(obj[i], elementKey)\n if encoded ~= nil then\n table.insert(items, encoded)\n end\n end\n if #items == 0 then\n return nil\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n local items = {}\n local fieldOrder = FIELD_ORDERS[key] or {}\n\n -- Phase 1: ordered keys\n for _, fieldName in ipairs(fieldOrder) do\n local v = obj[fieldName]\n if v ~= nil then\n local encoded = encodeJson(v, fieldName)\n if encoded ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\": ' .. encoded)\n end\n end\n end\n\n -- Phase 2: remaining keys\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then\n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local encoded = encodeJson(v, keyStr)\n if encoded ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. encoded)\n end\n end\n end\n\n if #items == 0 then\n return nil\n end\n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal function filterNullValues(obj)\n local filtered = {}\n for k, v in pairs(obj or {}) do\n if v ~= nil and v ~= \"\" and v ~= \"null\" then\n if type(v) == \"table\" then\n local nested = filterNullValues(v)\n if next(nested) then\n filtered[k] = nested\n end\n else\n filtered[k] = v\n end\n end\n end\n return filtered\nend\n\nlocal function isArrayTable(obj)\n if type(obj) ~= \"table\" then\n return false\n end\n local count = 0\n for k, _ in pairs(obj) do\n if type(k) ~= \"number\" then\n return false\n end\n count = count + 1\n end\n return count == #obj\nend\n\nlocal function applyFieldOrdering(obj, fieldOrder)\n if isArrayTable(obj) then\n local out = {}\n for i = 1, #obj do\n out[i] = obj[i]\n end\n return out\n end\n local ordered = {}\n for _, key in ipairs(fieldOrder or {}) do\n if obj[key] ~= nil then\n if type(obj[key]) == \"table\" then\n ordered[key] = applyFieldOrdering(obj[key], FIELD_ORDERS[key])\n else\n ordered[key] = obj[key]\n end\n end\n end\n for key, value in pairs(obj or {}) do\n local found = false\n for _, orderedKey in ipairs(fieldOrder or {}) do\n if orderedKey == key then\n found = true\n break\n end\n end\n if not found then\n if type(value) == \"table\" then\n ordered[key] = applyFieldOrdering(value, FIELD_ORDERS[key])\n else\n ordered[key] = value\n end\n end\n end\n return ordered\nend\n\nlocal function collectUnmapped(source, target, ignoreKeys)\n for key, value in pairs(source or {}) do\n if not (ignoreKeys and ignoreKeys[key]) then\n if type(value) == \"table\" then\n local nested = {}\n collectUnmapped(value, nested, ignoreKeys)\n if next(nested) then\n target[key] = nested\n end\n else\n target[key] = value\n end\n end\n end\nend\n\nlocal function getSeverityInfo(severity)\n local normalized = severity and severity:upper() or \"\"\n local info = Mapping.SEVERITY[normalized]\n if info then\n return info[1], info[2]\n end\n return 99, \"Unknown\" -- Default for unmapped severities (matches Python)\nend\n\nlocal function getEventType(logName)\n if not logName then\n return \"unknown\"\n end\n\n if string.find(logName, \"activity\") then\n return \"admin_activity\"\n elseif string.find(logName, \"data_access\") then\n return \"data_access\"\n elseif string.find(logName, \"system_event\") then\n return \"system_event\"\n elseif string.find(logName, \"policy\") then\n return \"policy_denied\"\n end\n\n return \"unknown\"\nend\n\nlocal function getActivityInfo(eventType)\n local info = Mapping.ACTIVITY[eventType]\n if info then\n return info[1], info[2]\n end\n return 99, \"Unknown\"\nend\n\nlocal function getTypeInfo(eventType)\n local info = Mapping.TYPE[eventType]\n if info then\n return info[1], info[2]\n end\n return 300499, \"Entity Management: Unknown\"\nend\n\nlocal function getClassInfo(eventType)\n local info = Mapping.CLASS[eventType]\n if info then\n return info[1], info[2]\n end\n return 3004, \"Entity Management\"\nend\n\nlocal function buildSource(event)\n local source = deepCopy(event, IGNORE_KEYS) or {}\n\n source.message = encodeJson(source, \"message\")\n\n -- Determine event type from logName\n local eventType = getEventType(source.logName)\n\n\n -- Get activity information\n local activity_id, activity_name = getActivityInfo(eventType)\n local type_uid, type_name = getTypeInfo(eventType)\n local class_uid, class_name = getClassInfo(eventType)\n\n -- Get severity information\n local severity_id, severity_label = getSeverityInfo(source.severity)\n\n -- Set base event properties\n source.activity_id = activity_id\n source.activity_name = activity_name\n source.category_uid = BaseEvent.CATEGORY_UID\n source.category_name = BaseEvent.CATEGORY_NAME\n source.class_uid = class_uid\n source.class_name = class_name\n \n source.severity_id = severity_id\n source.type_uid = type_uid\n --source.type_name = type_name\n\n -- Set data source\n source.dataSource = {\n vendor = DataSourceEvent.VENDOR,\n name = DataSourceEvent.NAME,\n category = DataSourceEvent.CATEGORY\n }\n\n -- Set metadata\n source.metadata = source.metadata or {}\n source.metadata.product = source.metadata.product or {}\n source.metadata.product.name = MetaDataEvent.PRODUCT_NAME\n source.metadata.product.vendor_name = MetaDataEvent.PRODUCT_VENDOR_NAME\n source.metadata.version = MetaDataEvent.VERSION\n\n -- Set site if available (siteId can be passed as parameter or extracted from log)\n source.site = source.site or {}\n source.site.id = source.site.id or \"\"\n\n -- Set event type\n source.event = {type = activity_name or \"\"}\n\n -- Map GCP audit log specific fields\n source.metadata.original_time = source.timestamp\n MAPPED_FIELDS[\"timestamp\"] = true\n source.metadata.logged_time = source.receiveTimestamp\n MAPPED_FIELDS[\"receiveTimestamp\"] = true\n source.metadata.log_name = source.logName\n MAPPED_FIELDS[\"logName\"] = true\n source.metadata.event_code = getValueByPath(source, \"operation.id\") or \"\"\n MAPPED_FIELDS[\"operation.id\"] = true\n source.metadata.uid = source.insertId\n MAPPED_FIELDS[\"insertId\"] = true\n -- Map actor and device information\n if source.protoPayload and source.protoPayload.authenticationInfo then\n source.actor = source.actor or {}\n source.actor.user = source.actor.user or {}\n source.actor.user.email_addr = source.protoPayload.authenticationInfo.principalEmail\n MAPPED_FIELDS[\"protoPayload.authenticationInfo.principalEmail\"] = true\n end\n\n if source.protoPayload and source.protoPayload.requestMetadata then\n source.device = source.device or {}\n source.device.ip = source.protoPayload.requestMetadata.callerIp\n MAPPED_FIELDS[\"protoPayload.requestMetadata.callerIp\"] = true\n end\n\n -- Map API information\n if source.protoPayload then\n source.api = source.api or {}\n source.api.service = source.api.service or {}\n source.api.service.name = source.protoPayload.serviceName\n MAPPED_FIELDS[\"protoPayload.serviceName\"] = true\n source.api.operation = getValueByPath(source, \"operation.producer\") or \"\"\n MAPPED_FIELDS[\"operation.producer\"] = true\n end\n\n -- Map resource information\n -- For policy denied events, use resource.name (not entity.name)\n -- For all other events, use entity.name (if resourceName exists)\n if source.protoPayload and source.protoPayload.resourceName then\n if eventType == \"policy_denied\" then\n source.resource = source.resource or {}\n source.resource.name = source.protoPayload.resourceName\n else\n source.entity = source.entity or {}\n source.entity.name = source.protoPayload.resourceName\n end\n MAPPED_FIELDS[\"protoPayload.resourceName\"] = true\n end\n\n -- Extract status fields from protoPayload.status (satisfies getpolicyDeniedOCSFMapping)\n if source.protoPayload and source.protoPayload.status then\n source.status_code = source.protoPayload.status.code\n source.status = source.protoPayload.status.message\n source.status_detail = source.protoPayload.status.details\n -- Mark status fields as mapped to avoid duplication\n MAPPED_FIELDS[\"protoPayload.status.code\"] = true\n MAPPED_FIELDS[\"protoPayload.status.message\"] = true\n MAPPED_FIELDS[\"protoPayload.status.details\"] = true\n end\n\n -- Set cloud provider\n source.cloud = source.cloud or {}\n source.cloud.provider = \"GCP\"\n\n\n -- Set time\n source.time = convertUtcToMilliseconds(source.timestamp)\n -- timestamp is already marked as mapped above\n\n -- Build message directly with ordering-aware encoder\n\n return source\nend\n\nlocal function cleanupEmptyNull(obj)\n\n if type(obj) ~= \"table\" then\n if type(obj) == \"string\" then\n local lower = obj:lower()\n if lower == \"\" or lower == \"null\" then\n return nil\n end\n end\n return obj\n end\n\n for key, value in pairs(obj) do\n local cleaned = cleanupEmptyNull(value)\n if cleaned == nil then\n obj[key] = nil\n else\n obj[key] = cleaned\n end\n end\n\n if next(obj) == nil then\n return nil\n end\n return obj\nend\n\nlocal function processEvent(event)\n if type(event) ~= \"table\" then\n return nil\n end\n\n -- reset mapped fields per event to avoid cross-event leakage\n MAPPED_FIELDS = {}\n\n local source = buildSource(event)\n\n local result = {}\n\n for _, mapping in ipairs(FIELD_MAPPINGS) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n MAPPED_FIELDS[mapping.source] = true\n end\n end\n\n -- Remove mapped fields from original event\n for key, _ in pairs(MAPPED_FIELDS) do\n setValueByPath(event, key, nil)\n end\n\n -- Collect remaining unmapped fields (fields not mapped to OCSF output)\n local unmapped = {}\n collectUnmapped(event, unmapped, IGNORE_KEYS)\n if next(unmapped) then\n result.unmapped = unmapped\n end\n\n if FEATURES.FLATTEN_EVENT_TYPE then\n if source and source.event then\n result['event.type'] = source.event.type\n end\n end\n\n if FEATURES.CLEANUP_EMPTY_NULL then\n cleanupEmptyNull(result)\n end\n\n return result\nend", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "F", - "harness_score": 25, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": false, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/metadata.yaml b/pipelines/community/transform_ocsf/gcp_audit_logs/metadata.yaml deleted file mode 100644 index 8ff5000..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/metadata.yaml +++ /dev/null @@ -1,47 +0,0 @@ -grade: - letter: F - score: 25 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Gcp Audit Logs. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: gcp - dataSource: Gcp Audit Logs - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "OAuth" - sample_record: "{\n \"kind\": \"admin#reports#activity\",\n \"id\": {\n \"time\": \"2026-04-20T03:32:52.967234+00:00\"\ - ,\n \"uniqueQualifier\": \"4878542320735016189\",\n \"applicationName\": \"admin\",\n \"\ - customerId\": \"C01NCC1701\"\n },\n \"etag\": \"\\\"d27c1b97272840a9b0d1738ada53d081\\\"\",\n \"\ - actor\": {\n \"email\": \"starfleet-admin@enterprise.starfleet.corp\",\n \"profileId\": \"328913592501844953\"\ - \n },\n \"ipAddress\": \"123.60.216.77\",\n \"events\": [\n {\n \"type\": \"USER_SETTINGS\"\ - ,\n \"name\": \"GRANT_ADMIN_ROLE\",\n \"parameters\": [\n {\n \"name\":\ - \ \"USER_EMAIL\",\n \"value\": \"jean.picard@starfleet.corp\"\n },\n {\n \ - \ \"name\": \"USER_NAME\",\n \"value\": \"Jean-Luc Picard\"\n },\n {\n\ - \ \"name\": \"SETTING_NAME\",\n \"value\": \"Calendar\"\n },\n {\n\ - \ \"name\": \"NEW_VALUE\",\n \"value\": \"ENABLED\"\n },\n {\n \ - \ \"name\": \"OLD_VALUE\",\n \"value\": \"DISABLED\"\n }\n ]\n }\n \ - \ ],\n \"ownerDomain\": \"starfleet.corp\"\n}" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: gcp, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: F - harness_score: 25 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/sample.json b/pipelines/community/transform_ocsf/gcp_audit_logs/sample.json deleted file mode 100644 index 8faddcf..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/sample.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "kind": "admin#reports#activity", - "id": { - "time": "2026-04-20T03:32:52.967234+00:00", - "uniqueQualifier": "4878542320735016189", - "applicationName": "admin", - "customerId": "C01NCC1701" - }, - "etag": "\"d27c1b97272840a9b0d1738ada53d081\"", - "actor": { - "email": "starfleet-admin@enterprise.starfleet.corp", - "profileId": "328913592501844953" - }, - "ipAddress": "123.60.216.77", - "events": [ - { - "type": "USER_SETTINGS", - "name": "GRANT_ADMIN_ROLE", - "parameters": [ - { - "name": "USER_EMAIL", - "value": "jean.picard@starfleet.corp" - }, - { - "name": "USER_NAME", - "value": "Jean-Luc Picard" - }, - { - "name": "SETTING_NAME", - "value": "Calendar" - }, - { - "name": "NEW_VALUE", - "value": "ENABLED" - }, - { - "name": "OLD_VALUE", - "value": "DISABLED" - } - ] - } - ], - "ownerDomain": "starfleet.corp" -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/serializer.lua b/pipelines/community/transform_ocsf/gcp_audit_logs/serializer.lua deleted file mode 100644 index 032bf88..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/serializer.lua +++ /dev/null @@ -1,623 +0,0 @@ --- GCP Audit Log OCSF 1.0.0 Schema Serializer -local FEATURES = { - FLATTEN_EVENT_TYPE = true, - CLEANUP_EMPTY_NULL = true, -} - -local BaseEvent = { - CATEGORY_UID = 3, - CATEGORY_NAME = "Identity & Access Management", - TIMEZONE_OFFSET = 0, - STATUS_ID = 0, - STATUS = "Unknown" -} - -local DataSourceEvent = { - VENDOR = "GCP", - NAME = "GCP Audit", - CATEGORY = "security" -} - -local MetaDataEvent = { - LOG_PROVIDER = "GCP Audit", - PRODUCT_LANG = "en", - PRODUCT_NAME = "GCP Audit", - PRODUCT_VENDOR_NAME = "GCP", - VERSION = "1.0.0" -} - -local Mapping = { - SEVERITY = { - ["INFO"] = {1, "Informational"}, - ["CRITICAL"] = {5, "Critical"} - }, - ACTIVITY = { - admin_activity = {99, "AdminActivity"}, - data_access = {99, "DataAccess"}, - system_event = {99, "SystemEvent"}, - policy_denied = {99, "PolicyDenied"} - }, - TYPE = { - admin_activity = {300499, "Entity Management: AdminActivity"}, - data_access = {300499, "Entity Management: DataAccess"}, - system_event = {300499, "Entity Management: SystemEvent"}, - policy_denied = {300599, "User Access Management: PolicyDenied"} - }, - CLASS = { - admin_activity = {3004, "Entity Management"}, - data_access = {3004, "Entity Management"}, - system_event = {3004, "Entity Management"}, - policy_denied = {3005, "User Access Management"} - } -} - -local MAPPED_FIELDS = {} - -local FIELD_ORDERS = { - root = { - "activity_id","activity_name","category_uid","category_name","class_uid","class_name","severity_id","type_uid","type_name","dataSource","site.id","metadata.product.name","metadata.product.vendor_name","metadata.version","actor.user.email_addr","device.ip","api.service.name","entity.name","resource.name","metadata.uid","metadata.original_time","metadata.logged_time","metadata.log_name","metadata.event_code","api.operation","cloud.provider","event.type","message","status_code","status","status_detail","time" - }, - metadata = {"original_time", "logged_time", "log_name", "event_code", "uid", "product", "version", "currentLocations", "deviceState", "@type", "violationReason", "intermediateServices", "securityPolicyInfo", "resourceNames", "ingressViolations", "vpcServiceControlsUniqueId"}, - product = {"name", "vendor_name"}, - dataSource = {"category", "name", "vendor"}, - site = {"id"}, - event = {"type"}, - message = {"protoPayload", "insertId", "resource", "timestamp", "severity", "logName", "operation", "receiveTimestamp"}, - protoPayload = {"@type", "status", "authenticationInfo", "requestMetadata", "serviceName", "methodName", "authorizationInfo", "resourceName", "resourceLocation", "request", "metadata"}, - status = {"code", "message", "details"}, - authenticationInfo = {"principalEmail"}, - requestMetadata = {"callerIp", "callerSuppliedUserAgent", "requestAttributes", "destinationAttributes"}, - requestAttributes = {"time", "reason", "auth"}, - authorizationInfo = {"resource", "permission", "granted", "resourceAttributes"}, - resource = {"type", "labels"}, - operation = {"id", "producer", "first"}, - labels = {"method", "service", "project_id", "location", "bucket_name", "subnetwork_name", "subnetwork_id"}, - request = {"@type"}, - preconditionFailure = {"@type", "violations"}, - details = {"@type", "violations"}, - violations = {"description", "type"}, - ingressViolations = {"targetResource", "servicePerimeter"}, - securityPolicyInfo = {"organizationId", "servicePerimeterName"}, -} - - -local FIELD_MAPPINGS = { - {source = "activity_id", target = "activity_id"}, - {source = "activity_name", target = "activity_name"}, - {source = "category_uid", target = "category_uid"}, - {source = "category_name", target = "category_name"}, - {source = "class_uid", target = "class_uid"}, - {source = "class_name", target = "class_name"}, - {source = "severity_id", target = "severity_id"}, - {source = "type_uid", target = "type_uid"}, - {source = "status_code", target = "status_code"}, - {source = "status", target = "status"}, - {source = "status_detail", target = "status_detail"}, - {source = "dataSource", target = "dataSource"}, - {source = "site.id", target = "site.id"}, - {source = "metadata.product.name", target = "metadata.product.name"}, - {source = "metadata.product.vendor_name", target = "metadata.product.vendor_name"}, - {source = "metadata.version", target = "metadata.version"}, - {source = "metadata.uid", target = "metadata.uid"}, - {source = "metadata.original_time", target = "metadata.original_time"}, - {source = "metadata.logged_time", target = "metadata.logged_time"}, - {source = "metadata.log_name", target = "metadata.log_name"}, - {source = "metadata.event_code", target = "metadata.event_code"}, - {source = "actor.user.email_addr", target = "actor.user.email_addr"}, - {source = "device.ip", target = "device.ip"}, - {source = "api.service.name", target = "api.service.name"}, - {source = "api.operation", target = "api.operation"}, - {source = "entity.name", target = "entity.name"}, - {source = "resource.name", target = "resource.name"}, - {source = "cloud.provider", target = "cloud.provider"}, - {source = "time", target = "time"}, - {source = "message", target = "message"} - --{source = "event.type", target = "event.type"}, - -} - -local IGNORE_KEYS = { - _ob = true, - message_id = true, -} - -local function split(str, delimiter) - if not str or str == "" then - return {} - end - local result = {} - for token in string.gmatch(str, "([^" .. delimiter .. "]+)") do - table.insert(result, token) - end - return result -end - -local function getValueByPath(obj, path) - local current = obj - for _, part in ipairs(split(path, ".")) do - - if type(current) ~= "table" then - return nil - end - - local key = tonumber(part) or part - - current = current[key] - end - return current -end - -local function setValueByPath(obj, path, value) - local parts = split(path, ".") - local current = obj - for i = 1, #parts - 1 do - local key = tonumber(parts[i]) or parts[i] - if current[key] == nil or type(current[key]) ~= "table" then - current[key] = {} - end - current = current[key] - end - local last = parts[#parts] - if value == nil then - current[last] = nil - else - current[last] = value - end -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - -local function convertUtcToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - -local function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" or obj == "" then - return nil - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray then - local items = {} - for i = 1, maxIndex do - local elementKey = key or tostring(i) - local encoded = encodeJson(obj[i], elementKey) - if encoded ~= nil then - table.insert(items, encoded) - end - end - if #items == 0 then - return nil - end - return "[" .. table.concat(items, ", ") .. "]" - else - local items = {} - local fieldOrder = FIELD_ORDERS[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrder) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '": ' .. encoded) - end - end - end - - -- Phase 2: remaining keys - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. encoded) - end - end - end - - if #items == 0 then - return nil - end - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local function filterNullValues(obj) - local filtered = {} - for k, v in pairs(obj or {}) do - if v ~= nil and v ~= "" and v ~= "null" then - if type(v) == "table" then - local nested = filterNullValues(v) - if next(nested) then - filtered[k] = nested - end - else - filtered[k] = v - end - end - end - return filtered -end - -local function isArrayTable(obj) - if type(obj) ~= "table" then - return false - end - local count = 0 - for k, _ in pairs(obj) do - if type(k) ~= "number" then - return false - end - count = count + 1 - end - return count == #obj -end - -local function applyFieldOrdering(obj, fieldOrder) - if isArrayTable(obj) then - local out = {} - for i = 1, #obj do - out[i] = obj[i] - end - return out - end - local ordered = {} - for _, key in ipairs(fieldOrder or {}) do - if obj[key] ~= nil then - if type(obj[key]) == "table" then - ordered[key] = applyFieldOrdering(obj[key], FIELD_ORDERS[key]) - else - ordered[key] = obj[key] - end - end - end - for key, value in pairs(obj or {}) do - local found = false - for _, orderedKey in ipairs(fieldOrder or {}) do - if orderedKey == key then - found = true - break - end - end - if not found then - if type(value) == "table" then - ordered[key] = applyFieldOrdering(value, FIELD_ORDERS[key]) - else - ordered[key] = value - end - end - end - return ordered -end - -local function collectUnmapped(source, target, ignoreKeys) - for key, value in pairs(source or {}) do - if not (ignoreKeys and ignoreKeys[key]) then - if type(value) == "table" then - local nested = {} - collectUnmapped(value, nested, ignoreKeys) - if next(nested) then - target[key] = nested - end - else - target[key] = value - end - end - end -end - -local function getSeverityInfo(severity) - local normalized = severity and severity:upper() or "" - local info = Mapping.SEVERITY[normalized] - if info then - return info[1], info[2] - end - return 99, "Unknown" -- Default for unmapped severities (matches Python) -end - -local function getEventType(logName) - if not logName then - return "unknown" - end - - if string.find(logName, "activity") then - return "admin_activity" - elseif string.find(logName, "data_access") then - return "data_access" - elseif string.find(logName, "system_event") then - return "system_event" - elseif string.find(logName, "policy") then - return "policy_denied" - end - - return "unknown" -end - -local function getActivityInfo(eventType) - local info = Mapping.ACTIVITY[eventType] - if info then - return info[1], info[2] - end - return 99, "Unknown" -end - -local function getTypeInfo(eventType) - local info = Mapping.TYPE[eventType] - if info then - return info[1], info[2] - end - return 300499, "Entity Management: Unknown" -end - -local function getClassInfo(eventType) - local info = Mapping.CLASS[eventType] - if info then - return info[1], info[2] - end - return 3004, "Entity Management" -end - -local function buildSource(event) - local source = deepCopy(event, IGNORE_KEYS) or {} - - source.message = encodeJson(source, "message") - - -- Determine event type from logName - local eventType = getEventType(source.logName) - - - -- Get activity information - local activity_id, activity_name = getActivityInfo(eventType) - local type_uid, type_name = getTypeInfo(eventType) - local class_uid, class_name = getClassInfo(eventType) - - -- Get severity information - local severity_id, severity_label = getSeverityInfo(source.severity) - - -- Set base event properties - source.activity_id = activity_id - source.activity_name = activity_name - source.category_uid = BaseEvent.CATEGORY_UID - source.category_name = BaseEvent.CATEGORY_NAME - source.class_uid = class_uid - source.class_name = class_name - - source.severity_id = severity_id - source.type_uid = type_uid - --source.type_name = type_name - - -- Set data source - source.dataSource = { - vendor = DataSourceEvent.VENDOR, - name = DataSourceEvent.NAME, - category = DataSourceEvent.CATEGORY - } - - -- Set metadata - source.metadata = source.metadata or {} - source.metadata.product = source.metadata.product or {} - source.metadata.product.name = MetaDataEvent.PRODUCT_NAME - source.metadata.product.vendor_name = MetaDataEvent.PRODUCT_VENDOR_NAME - source.metadata.version = MetaDataEvent.VERSION - - -- Set site if available (siteId can be passed as parameter or extracted from log) - source.site = source.site or {} - source.site.id = source.site.id or "" - - -- Set event type - source.event = {type = activity_name or ""} - - -- Map GCP audit log specific fields - source.metadata.original_time = source.timestamp - MAPPED_FIELDS["timestamp"] = true - source.metadata.logged_time = source.receiveTimestamp - MAPPED_FIELDS["receiveTimestamp"] = true - source.metadata.log_name = source.logName - MAPPED_FIELDS["logName"] = true - source.metadata.event_code = getValueByPath(source, "operation.id") or "" - MAPPED_FIELDS["operation.id"] = true - source.metadata.uid = source.insertId - MAPPED_FIELDS["insertId"] = true - -- Map actor and device information - if source.protoPayload and source.protoPayload.authenticationInfo then - source.actor = source.actor or {} - source.actor.user = source.actor.user or {} - source.actor.user.email_addr = source.protoPayload.authenticationInfo.principalEmail - MAPPED_FIELDS["protoPayload.authenticationInfo.principalEmail"] = true - end - - if source.protoPayload and source.protoPayload.requestMetadata then - source.device = source.device or {} - source.device.ip = source.protoPayload.requestMetadata.callerIp - MAPPED_FIELDS["protoPayload.requestMetadata.callerIp"] = true - end - - -- Map API information - if source.protoPayload then - source.api = source.api or {} - source.api.service = source.api.service or {} - source.api.service.name = source.protoPayload.serviceName - MAPPED_FIELDS["protoPayload.serviceName"] = true - source.api.operation = getValueByPath(source, "operation.producer") or "" - MAPPED_FIELDS["operation.producer"] = true - end - - -- Map resource information - -- For policy denied events, use resource.name (not entity.name) - -- For all other events, use entity.name (if resourceName exists) - if source.protoPayload and source.protoPayload.resourceName then - if eventType == "policy_denied" then - source.resource = source.resource or {} - source.resource.name = source.protoPayload.resourceName - else - source.entity = source.entity or {} - source.entity.name = source.protoPayload.resourceName - end - MAPPED_FIELDS["protoPayload.resourceName"] = true - end - - -- Extract status fields from protoPayload.status (satisfies getpolicyDeniedOCSFMapping) - if source.protoPayload and source.protoPayload.status then - source.status_code = source.protoPayload.status.code - source.status = source.protoPayload.status.message - source.status_detail = source.protoPayload.status.details - -- Mark status fields as mapped to avoid duplication - MAPPED_FIELDS["protoPayload.status.code"] = true - MAPPED_FIELDS["protoPayload.status.message"] = true - MAPPED_FIELDS["protoPayload.status.details"] = true - end - - -- Set cloud provider - source.cloud = source.cloud or {} - source.cloud.provider = "GCP" - - - -- Set time - source.time = convertUtcToMilliseconds(source.timestamp) - -- timestamp is already marked as mapped above - - -- Build message directly with ordering-aware encoder - - return source -end - -local function cleanupEmptyNull(obj) - - if type(obj) ~= "table" then - if type(obj) == "string" then - local lower = obj:lower() - if lower == "" or lower == "null" then - return nil - end - end - return obj - end - - for key, value in pairs(obj) do - local cleaned = cleanupEmptyNull(value) - if cleaned == nil then - obj[key] = nil - else - obj[key] = cleaned - end - end - - if next(obj) == nil then - return nil - end - return obj -end - -local function processEvent(event) - if type(event) ~= "table" then - return nil - end - - -- reset mapped fields per event to avoid cross-event leakage - MAPPED_FIELDS = {} - - local source = buildSource(event) - - local result = {} - - for _, mapping in ipairs(FIELD_MAPPINGS) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - MAPPED_FIELDS[mapping.source] = true - end - end - - -- Remove mapped fields from original event - for key, _ in pairs(MAPPED_FIELDS) do - setValueByPath(event, key, nil) - end - - -- Collect remaining unmapped fields (fields not mapped to OCSF output) - local unmapped = {} - collectUnmapped(event, unmapped, IGNORE_KEYS) - if next(unmapped) then - result.unmapped = unmapped - end - - if FEATURES.FLATTEN_EVENT_TYPE then - if source and source.event then - result['event.type'] = source.event.type - end - end - - if FEATURES.CLEANUP_EMPTY_NULL then - cleanupEmptyNull(result) - end - - return result -end \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/microsoft_365/metadata.yaml b/pipelines/community/transform_ocsf/microsoft_365/metadata.yaml deleted file mode 100644 index 642c3ce..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/metadata.yaml +++ /dev/null @@ -1,52 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Microsoft 365. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: microsoft - dataSource: Microsoft 365 - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "OAuth" - sample_record: "{\n \"id\": \"cb1628d2-7c07-4435-9f4e-87fd44024d05\",\n \"azureSubscriptionId\": \"\ - 6deb1e8b-896d-484a-a649-aeb6aebfeaf9\",\n \"azureTenantId\": \"8c7e1716-01d8-4aeb-ac89-c691e8957e6f\"\ - ,\n \"activityGroupName\": \"Suspicious email forwarding\",\n \"assignedTo\": \"unassigned\",\n\ - \ \"category\": \"MaliciousEmail\",\n \"closedDateTime\": \"2026-04-20T13:40:52.634172Z\",\n \"\ - cloudAppStates\": [\n {\n \"destinationServiceIp\": \"135.249.79.95\",\n \"destinationServiceName\"\ - : \"SharePoint Online\",\n \"riskScore\": \"66\"\n }\n ],\n \"comments\": [\n \"Alert\ - \ generated at 2026-04-20 03:33:52\",\n \"Escalated to security team\"\n ],\n \"confidence\"\ - : 97,\n \"createdDateTime\": \"2026-04-20T03:33:52.634172Z\",\n \"description\": \"Microsoft 365\ - \ security alert: Suspicious email forwarding. This alert indicates potential maliciousemail activity\ - \ detected in your environment.\",\n \"detectionIds\": [\n \"de01dafc-efd2-47a5-9d71-9ca354ae2bc5\"\ - \n ],\n \"eventDateTime\": \"2026-04-20T03:36:52.634172Z\",\n \"feedback\": \"falsePositive\",\n\ - \ \"incidentIds\": [],\n \"lastEventDateTime\": \"2026-04-20T03:54:52.634172Z\",\n \"lastModifiedDateTime\"\ - : \"2026-04-20T03:40:52.634172Z\",\n \"malwareStates\": [],\n \"networkConnections\": [\n {\n\ - \ \"applicationName\": \"chrome.exe\",\n \"destinationAddress\": \"199.227.89.52\",\n \ - \ \"destinationDomain\": \"graph.microsoft.com\",\n \"destinationPort\": \"8080\",\n \ - \ \"destinationUrl\": \"https://domain-62.com/api/data\",\n \"direction\": \"Outbound\",\n \ - \ \"domainRegisteredDateTime\": \"2025-07-18T03:40:52.634250Z\",\n " - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: microsoft, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/microsoft_365/microsoft_365.json b/pipelines/community/transform_ocsf/microsoft_365/microsoft_365.json deleted file mode 100644 index 42c323d..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/microsoft_365.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Microsoft 365", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Microsoft 365. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "microsoft", - "source_name": "microsoft_365", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "microsoft-365-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- Microsoft 365 (O365) OCSF 1.0.0 parser (ported from Python Parsers/microsoft)\n\n-- Helpers\n\nlocal function convertUtcToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\nlocal function split(str, delimiter)\n local result = {}\n local s = tostring(str or \"\")\n local escaped = delimiter:gsub(\"[%.%+%*%?%^%$%(%)%[%]%%]\", \"%%%1\")\n local pattern = \"([^\" .. escaped .. \"]+)\"\n for token in s:gmatch(pattern) do table.insert(result, token) end\n if #result == 0 and #s > 0 then table.insert(result, s) end\n return result\nend\n\nlocal function getByPath(obj, keys)\n local cur = obj\n for _, k in ipairs(keys) do\n if cur ~= nil and type(cur) == \"table\" then cur = cur[k] else return nil end\n end\n return cur\nend\n\nlocal function setByDottedPath(t, dotted, value)\n local keys = split(dotted, \".\")\n local cur = t\n for i=1,#keys-1 do\n local k = keys[i]\n if type(cur[k]) ~= \"table\" then cur[k] = {} end\n cur = cur[k]\n end\n cur[keys[#keys]] = value\nend\n\nlocal function flatten_table(tbl, prefix, out)\n out = out or {}\n prefix = prefix or \"\"\n\n for k, v in pairs(tbl) do\n local key\n\n -- ARRAY HANDLING (0-based index)\n if type(k) == \"number\" then\n key = string.format(\"%s[%d]\", prefix, k - 1)\n\n -- NORMAL OBJECT KEY\n else\n if prefix ~= \"\" then\n key = prefix .. \".\" .. k\n else\n key = k\n end\n end\n\n -- recurse\n if type(v) == \"table\" then\n flatten_table(v, key, out)\n else\n out[key] = v\n end\n end\n\n return out\nend\n\nlocal function apply_mapping(event, mapping)\n local out = {}\n\n -- normal mapping (existing behaviour)\n for src, dst in pairs(mapping) do\n local val = getByPath(event, split(src, \".\"))\n if val ~= nil then out[dst] = val end\n end\n\n -- NEW: auto-capture unmapped fields\n local flat_event = flatten_table(event)\n for src, val in pairs(flat_event) do\n if mapping[src] == nil then\n out[\"unmapped.\" .. src] = val\n end\n end\n\n return out\nend\n\nlocal function build_nested(flat)\n local nested = {}\n for k, v in pairs(flat) do setByDottedPath(nested, k, v) end\n return nested\nend\n\nlocal function cloneTable(src)\n local dst = {}\n for k,v in pairs(src or {}) do dst[k] = v end\n return dst\nend\n\n-- Simple email detection\nlocal function is_email(s)\n if type(s) ~= \"string\" then return false end\n return s:find(\"@\") ~= nil\nend\n\n-- Common + synthetic field helpers\nlocal function get_status_default_ocsf_mapping(status)\n if status == \"Success\" then return 1\n elseif status == \"Failure\" then return 2\n else return 99 end\nend\n\nlocal function get_device_os_type(platform)\n local mapping = { Win = 100, Android = 201, [\"iOS\"] = 301 }\n if type(platform) ~= \"string\" then return 99 end\n return mapping[platform] or 99\nend\n\nlocal function get_actor_type_details(user_type)\n local map = {\n [0] = { type = \"User\", id = 1 },\n [2] = { type = \"Admin\", id = 2 },\n [4] = { type = \"System\", id = 4 },\n }\n local rec = map[user_type]\n if rec then return rec.type, rec.id else return \"Other\", 99 end\nend\n\nlocal function get_state_id(event_type)\n if type(event_type) == \"string\" and event_type:lower() == \"dlprulematch\" then return 1 else return 0 end\nend\n\nlocal function extract_policy_details(log)\n local analytic_id, analytic_name = nil, nil\n local pd = log[\"PolicyDetails\"]\n if type(pd) == \"table\" and #pd > 0 and type(pd[1]) == \"table\" then\n if pd[1][\"PolicyId\"] then analytic_id = pd[1][\"PolicyId\"] end\n if pd[1][\"PolicyName\"] then analytic_name = pd[1][\"PolicyName\"] end\n end\n return analytic_id, analytic_name\nend\n\nlocal function get_device_property(log)\n local device_props = log[\"DeviceProperties\"]\n if type(device_props) == \"table\" then\n for _, dp in ipairs(device_props) do\n if type(dp) == \"table\" then\n if dp[\"Name\"] == \"OS\" then\n log[\"device_os_type\"] = dp[\"Value\"]\n log[\"device_os_type_id\"] = (dp[\"Value\"] == \"Linux\" and 200) or (dp[\"Value\"] == \"Windows\" and 100) or 99\n elseif dp[\"Name\"] == \"SessionId\" then\n log[\"actor_session_uid\"] = dp[\"Value\"]\n elseif dp[\"Name\"] == \"IsCompliantAndManaged\" then\n log[\"device_is_compliant\"] = dp[\"Value\"]\n log[\"device_is_managed\"] = dp[\"Value\"]\n end\n end\n end\n end\n return log\nend\n\nlocal function get_resource_data(log)\n local out = {}\n local mprops = log[\"ModifiedProperties\"]\n if type(mprops) == \"table\" then\n for _, obj in ipairs(mprops) do\n if type(obj) == \"table\" and obj[\"Name\"] then\n out[obj[\"Name\"]] = { NewValue = obj[\"NewValue\"], OldValue = obj[\"OldValue\"] }\n end\n end\n end\n return out\nend\n\nlocal function get_mgmt_observables(log)\n local obs = {}\n local sp = log[\"SharePointMetaData\"]\n if type(sp) == \"table\" then\n if sp[\"FileName\"] then table.insert(obs, { type_id = 7, type = \"FileName\", name = \"process.file.name\", value = sp[\"FileName\"] }) end\n if sp[\"From\"] then table.insert(obs, { type_id = 4, type = \"User Name\", name = \"unmapped.sharePointMetaData.from\", value = sp[\"From\"] }) end\n end\n return obs\nend\n\n-- Synthetic for Graph API\nlocal function set_graph_synthetic_fields(log, site_id)\n log[\"category_uid\"] = 2\n log[\"class_uid\"] = 2001\n log[\"class_name\"] = \"Security Finding\"\n log[\"type_uid\"] = 200199\n log[\"type_name\"] = \"Security Finding: Other\"\n log[\"OCSF_version\"] = \"1.0.0\"\n log[\"category_name\"] = \"Findings\"\n log[\"activity_id\"] = 99\n log[\"dataSource\"] = { name = \"Microsoft O365\", category = \"security\", vendor = \"Microsoft\" }\n log[\"event\"] = { type = log[\"status\"] }\n log[\"cloud\"] = { provider = \"Microsoft Azure\" }\n if site_id then log[\"site\"] = { id = site_id } end\n -- observables\n local observables = {}\n if log[\"azureTenantId\"] then table.insert(observables, { type_id = 10, type = \"Resource UID\", name = \"unmapped.azureTenantId\", value = log[\"azureTenantId\"] }) end\n log[\"observables\"] = observables\n if type(log[\"sourceMaterials\"]) == \"table\" then log[\"sourceMaterials\"] = log[\"sourceMaterials\"][1] end\n return log\nend\n\n-- Severity for Mgmt\nlocal function get_severity_id(event_type)\n local m = {\n dlprulematch = 1, dlpruleundo = 0, [\"update user\"] = 0, userloggedin = 0, [\"add group\"] = 0,\n [\"add member to group\"] = 0, [\"delete group\"] = 0, [\"update group\"] = 0, [\"remove member from group\"] = 0,\n [\"add user\"] = 0, [\"reset user password\"] = 0, [\"delete user\"] = 0, addedtogroup = 0, signinevent = 0,\n }\n return m[(event_type or \"\"):lower()] or 0\nend\n\nlocal function get_mgmt_log_mapping_fields(event_type)\n local map = {\n [\"dlprulematch\"] = {\n activity = { name = \"Create\", id = 99 }, class = { name = \"Security Finding\", id = 2001 }, category = { name = \"Findings\", id = 2 },\n },\n [\"dlpruleundo\"] = {\n activity = { name = \"DLPRuleUndo\", id = 99 }, class = { name = \"Security Finding\", id = 2001 }, category = { name = \"Findings\", id = 2 },\n },\n [\"update user\"] = {\n activity = { name = \"Update user\", id = 99 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"userloggedin\"] = {\n activity = { name = \"Logon\", id = 1 }, class = { name = \"Authentication\", id = 3002 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"add group\"] = {\n activity = { name = \"Add group\", id = 99 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"add member to group\"] = {\n activity = { name = \"Add User\", id = 3 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"delete group\"] = {\n activity = { name = \"Delete group\", id = 99 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"update group\"] = {\n activity = { name = \"Update group.\", id = 99 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"remove member from group\"] = {\n activity = { name = \"Remove User\", id = 4 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"add user\"] = {\n activity = { name = \"Create\", id = 1 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"reset user password\"] = {\n activity = { name = \"Password Reset\", id = 4 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"delete user\"] = {\n activity = { name = \"Delete\", id = 6 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"addedtogroup\"] = {\n activity = { name = \"Add User\", id = 3 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"signinevent\"] = {\n activity = { name = \"Logon\", id = 1 }, class = { name = \"Authentication\", id = 3002 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n }\n local m = map[(event_type or \"\"):lower()] or { activity = { name = \"Other\", id = 99 }, class = { name = \"Base Event\", id = 0}, category = { name = \"Uncategorized\", id = 0 } }\n local class_name, class_id = m.class.name, m.class.id\n local activity_name, activity_id = m.activity.name, m.activity.id\n local type_name, type_id = string.format(\"%s: %s\", class_name, activity_name), (class_id * 100) + activity_id\n local category_name, category_id = m.category.name, m.category.id\n return activity_name, activity_id, class_name, class_id, type_name, type_id, category_name, category_id\nend\n\n-- Mapping tables from python\nlocal function get_default_graph_ocsf_mapping()\n return {\n [\"activityGroupName\"] = \"resources.group_name\",\n [\"category\"] = \"finding.types\",\n [\"closedDateTime\"] = \"end_time_dt\",\n [\"confidence\"] = \"confidence\",\n [\"createdDateTime\"] = \"metadata.original_time\",\n [\"eventDateTime\"] = \"start_time_dt\",\n [\"id\"] = \"metadata.uid\",\n [\"lastModifiedDateTime\"] = \"finding.modified_time\",\n [\"sourceMaterials\"] = \"finding.src_url\",\n [\"status\"] = \"activity_name\",\n [\"vendorInformation.provider\"] = \"metadata.product.name\",\n [\"vendorInformation.providerVersion\"] = \"metadata.product.version\",\n [\"vendorInformation.vendor\"] = \"metadata.product.vendor_name\",\n [\"lastEventDateTime\"] = \"end_time\",\n [\"title\"] = \"finding.title\",\n [\"category_uid\"] = \"category_uid\",\n [\"class_uid\"] = \"class_uid\",\n [\"type_uid\"] = \"type_uid\",\n [\"category_name\"] = \"category_name\",\n [\"class_name\"] = \"class_name\",\n [\"type_name\"] = \"type_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"cloud.provider\"] = \"cloud.provider\",\n [\"OCSF_version\"] = \"metadata.version\",\n [\"observables\"] = \"observables\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"event.type\"] = \"event.type\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n }\nend\n\nlocal function generic_mgmt_mapping()\n return {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n }\nend\n \nlocal function get_mgmt_default_mapping(event_type)\n -- ported from python OCSFMapping.get_mgmt_default_mapping\n local default = {\n [\"dlprulematch\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"finding.title\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"IncidentId\"] = \"finding.uid\",\n [\"SharePointMetaData.FileID\"] = \"process.file.uid\",\n [\"SharePointMetaData.FileName\"] = \"process.file.name\",\n [\"SharePointMetaData.FileOwner\"] = \"process.file.owner.uid\",\n [\"SharePointMetaData.FilePathUrl\"] = \"process.file.path\",\n [\"SharePointMetaData.FileSize\"] = \"process.file.size\",\n [\"SharePointMetaData.From\"] = \"resources.data.from\",\n [\"SharePointMetaData.IsViewableByExternalUsers\"] = \"resources.data.is_viewable_by_external_users\",\n [\"SharePointMetaData.IsVisibleOnlyToOdbOwner\"] = \"resources.data.is_visible_only_to_Odb_owner\",\n [\"SharePointMetaData.ItemCreationTime\"] = \"finding.created_time\",\n [\"SharePointMetaData.ItemLastModifiedTime\"] = \"finding.modified_time\",\n [\"SharePointMetaData.ItemLastSharedTime\"] = \"resources.data.item_last_shared_time\",\n [\"SharePointMetaData.SensitivityLabelIds\"] = \"resources.data.sensitivity_label_ids\",\n [\"SharePointMetaData.SharedBy\"] = \"resources.data.shared_by\",\n [\"SharePointMetaData.SiteAdmin\"] = \"resources.data.site_admin\",\n [\"SharePointMetaData.SiteCollectionGuid\"] = \"resources.data.site_collection_guid\",\n [\"SharePointMetaData.SiteCollectionUrl\"] = \"resources.data.site_collection_url\",\n [\"SharePointMetaData.UniqueID\"] = \"resources.data.unique_id\",\n [\"state_id\"] = \"state_id\",\n [\"status_id\"] = \"status_id\",\n [\"analytic.uid\"] = \"analytic.uid\",\n [\"analytic.name\"] = \"analytic.name\",\n [\"analytic_type_id\"] = \"analytic.type_id\",\n [\"process_file_owner_type_id\"] = \"process.file.owner.type_id\",\n },\n [\"dlpruleundo\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"finding.title\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"IncidentId\"] = \"finding.uid\",\n [\"SharePointMetaData.FileID\"] = \"process.file.uid\",\n [\"SharePointMetaData.FileName\"] = \"process.file.name\",\n [\"SharePointMetaData.FileOwner\"] = \"process.file.owner.uid\",\n [\"SharePointMetaData.FilePathUrl\"] = \"process.file.path\",\n [\"SharePointMetaData.FileSize\"] = \"process.file.size\",\n [\"SharePointMetaData.From\"] = \"resources.data.from\",\n [\"SharePointMetaData.IsViewableByExternalUsers\"] = \"resources.data.is_viewable_by_external_users\",\n [\"SharePointMetaData.IsVisibleOnlyToOdbOwner\"] = \"resources.data.is_visible_only_to_Odb_owner\",\n [\"SharePointMetaData.ItemCreationTime\"] = \"finding.created_time\",\n [\"SharePointMetaData.ItemLastModifiedTime\"] = \"finding.modified_time\",\n [\"SharePointMetaData.ItemLastSharedTime\"] = \"resources.data.item_last_shared_time\",\n [\"SharePointMetaData.SensitivityLabelIds\"] = \"resources.data.sensitivity_label_ids\",\n [\"SharePointMetaData.SharedBy\"] = \"resources.data.shared_by\",\n [\"SharePointMetaData.SiteAdmin\"] = \"resources.data.site_admin\",\n [\"SharePointMetaData.SiteCollectionGuid\"] = \"resources.data.site_collection_guid\",\n [\"SharePointMetaData.SiteCollectionUrl\"] = \"resources.data.site_collection_url\",\n [\"SharePointMetaData.UniqueID\"] = \"resources.data.unique_id\",\n [\"analytic.uid\"] = \"analytic.uid\",\n [\"analytic.name\"] = \"analytic.name\",\n [\"analytic_type_id\"] = \"analytic.type_id\",\n [\"process_file_owner_type_id\"] = \"process.file.owner.type_id\",\n [\"state_id\"] = \"state_id\",\n },\n [\"update user\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"activity_name\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"userloggedin\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ClientIP\"] = \"device.ip\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"ActorIpAddress\"] = \"src_endpoint.ip\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"ApplicationId\"] = \"dst_endpoint.uid\",\n [\"ErrorNumber\"] = \"api.response.code\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"device_os_type\"] = \"device.os.type\",\n [\"device_os_type_id\"] = \"device.os.type_id\",\n [\"actor_session_uid\"] = \"actor.session.uid\",\n [\"device_is_compliant\"] = \"device.is_compliant\",\n [\"device_is_managed\"] = \"device.is_managed\",\n },\n [\"add group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"activity_name\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"add member to group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"delete group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"update group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"activity_name\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"remove member from group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"add user\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"reset user password\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"delete user\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.UserKey\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"addedtogroup\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ClientIP\"] = \"device.ip\",\n [\"CorrelationId\"] = \"metadata.correlation_uid\",\n [\"EventSource\"] = \"resource.name\",\n [\"EventData\"] = \"status_detail\",\n [\"TargetUserOrGroupType\"] = \"user.groups.type\",\n [\"TargetUserOrGroupName\"] = \"user.groups.name\",\n [\"AppAccessContext.CorrelationId\"] = \"actor.idp.uid\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n },\n [\"signinevent\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ClientIP\"] = \"src_endpoint.ip\",\n [\"CorrelationId\"] = \"metadata.correlation_uid\",\n [\"EventSource\"] = \"service.name\",\n [\"IsManagedDevice\"] = \"device.is_managed\",\n [\"Platform\"] = \"device.os.type\",\n [\"UserAgent\"] = \"http_request.user_agent\",\n [\"DeviceDisplayName\"] = \"device.ip\",\n [\"AppAccessContext.AADSessionId\"] = \"actor.session.uid\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"device_os_type_id\"] = \"device.os.type_id\",\n },\n }\n local et = (event_type or \"\"):lower()\n return default[et] or generic_mgmt_mapping()\nend\n\nlocal function common_mapping()\n return {\n [\"category_name\"] = \"category_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"class_uid\"] = \"class_uid\",\n [\"severity_id\"] = \"severity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"type_uid\"] = \"type_uid\",\n [\"product_vendor_name\"] = \"metadata.product.vendor_name\",\n [\"product_name\"] = \"metadata.product.name\",\n [\"OCSF_version\"] = \"metadata.version\",\n [\"observables\"] = \"observables\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"event.type\"] = \"event.type\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"message\"] = \"message\",\n [\"class_name\"] = \"class_name\",\n [\"type_name\"] = \"type_name\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"status_id\"] = \"status_id\",\n }\nend\n\n-- Mgmt synthetic enrichment\nlocal function set_mgmt_synthetic_fields(log, site_id)\n log[\"product_vendor_name\"] = \"Microsoft\"\n log[\"OCSF_version\"] = \"1.0.0\"\n local event_type = tostring(log[\"Operation\"] or \"Other\")\n local event_type_dup = event_type:gsub(\"%.\", \"\")\n\n log[\"severity_id\"] = get_severity_id(event_type_dup)\n local activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid = get_mgmt_log_mapping_fields(event_type_dup)\n log[\"activity_name\"], log[\"activity_id\"], log[\"class_name\"], log[\"class_uid\"], log[\"type_name\"], log[\"type_uid\"], log[\"category_name\"], log[\"category_uid\"] = activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid\n\n log[\"status_id\"] = get_status_default_ocsf_mapping(log[\"ResultStatus\"])\n\n if site_id then log[\"site\"] = { id = site_id } end\n\n if log[\"activity_id\"] == 99 then\n log[\"event\"] = { type = event_type }\n log[\"activity_name\"] = event_type\n else\n log[\"event\"] = { type = log[\"activity_name\"] }\n end\n\n log[\"dataSource\"] = { name = \"Microsoft O365\", category = \"security\", vendor = \"Microsoft\" }\n\n if event_type_dup:lower() == \"dlprulematch\" or event_type_dup:lower() == \"dlpruleundo\" then\n log[\"analytic_type_id\"] = 1\n log[\"process_file_owner_type_id\"] = 99\n log[\"observables\"] = get_mgmt_observables(log)\n log[\"state_id\"] = get_state_id(event_type_dup)\n local aid, aname = extract_policy_details(log)\n log[\"analytic\"] = { uid = aid, name = aname }\n else\n -- set actor values\n if (event_type_dup:lower() == \"userloggedin\") then\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = \"User\", 1\n end\n local userId = log[\"UserId\"]\n if is_email(userId) then\n log[\"actor_user_email_addr\"] = userId\n elseif type(userId) == \"string\" and userId ~= \"\" then\n log[\"actor_user_uid\"] = userId\n end\n local actors = log[\"Actor\"]\n if type(actors) == \"table\" and #actors > 0 then\n for _, a in ipairs(actors) do\n if type(a) == \"table\" then\n local id = a[\"ID\"]\n if is_email(id) then\n log[\"actor_user_email_addr\"] = id\n local t, tid = get_actor_type_details(a[\"Type\"])\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = t, tid\n break\n elseif type(id) == \"string\" and id:match(\"-\") then\n log[\"actor_user_uid\"] = id\n local t, tid = get_actor_type_details(a[\"Type\"])\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = t, tid\n break\n end\n end\n end\n else\n local t, tid = get_actor_type_details(log[\"UserType\"])\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = t, tid\n end\n\n -- set user target data\n local target = log[\"Target\"]\n if type(target) == \"table\" and #target > 0 and type(target[1]) == \"table\" then\n local tid = target[1][\"ID\"]\n if is_email(tid) then log[\"user_email_addr\"] = tid else log[\"user_uid\"] = tid end\n local t, tidv = get_actor_type_details(target[1][\"Type\"])\n log[\"user_type\"], log[\"user_type_id\"] = t, tidv\n end\n end\n\n if log[\"UserKey\"] == \"Not Available\" or log[\"UserKey\"] == \"NA\" then log[\"UserKey\"] = \"\" end\n\n local etl = event_type_dup:lower()\n if etl == \"add group\" or etl == \"add member to group\" or etl == \"delete group\" or etl == \"update group\" or etl == \"remove member from group\" then\n log[\"resource\"] = { data = get_resource_data(log) }\n end\n\n if etl == \"signinevent\" then\n log[\"device_os_type_id\"] = get_device_os_type(log[\"Platform\"])\n end\n\n if etl == \"userloggedin\" then\n log = get_device_property(log)\n end\n\n return log\nend\n\n-- Parsers\nlocal function default_graph_api_parser(alertLog)\n return build_nested(apply_mapping(alertLog, get_default_graph_ocsf_mapping()))\nend\n\nlocal function default_mgmt_api_parser(log)\n local event_type = tostring(log[\"Operation\"] or \"Other\"):gsub(\"%.\", \"\")\n local mapping = get_mgmt_default_mapping(event_type)\n -- merge with common mapping\n for src, dst in pairs(common_mapping()) do mapping[src] = dst end\n local flat = apply_mapping(log, mapping)\n return build_nested(flat)\nend\n\n-- API detection: Graph has createdDateTime, vendorInformation; Mgmt has CreationTime/Operation\nlocal function detect_api(event)\n if event[\"createdDateTime\"] or getByPath(event, {\"vendorInformation\"}) then return \"Graph\" end\n return \"Mgmt\"\nend\n\n-- Helper: Encode Lua table to JSON string with field ordering\nlocal function encodeJson(obj, fieldOrder, key)\n if obj == nil then\n\t return \"null\"\n elseif type(obj) == \"boolean\" then\n\t return tostring(obj)\n elseif type(obj) == \"number\" then\n\t return tostring(obj)\n elseif type(obj) == \"string\" then\n\t return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n\t local isArray = true\n\t local maxIndex = 0\n\t for k, v in pairs(obj) do\n\t\t if type(k) ~= \"number\" then\n\t\t\t isArray = false\n\t\t\t break\n\t\t end\n\t\t maxIndex = math.max(maxIndex, k)\n\t end\n\t if isArray then\n\t\t local items = {}\n\t\t for i = 1, maxIndex do\n\t\t\t table.insert(items, encodeJson(obj[i], fieldOrder, key))\n\t\t end\n\t\t return \"[\" .. table.concat(items, \",\") .. \"]\"\n\t else\n\t\t local items = {}\n\t\t local fieldOrdering = fieldOrder[key] or {}\n\t\t \n\t\t -- Phase 1: ordered keys\n\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t local v = obj[fieldName]\n\t\t\t if v ~= nil then\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, fieldName)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t -- Phase 2: remaining keys (not in fieldOrder)\n\t\t for k, v in pairs(obj) do\n\t\t\t local found = false\n\t\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t\t if k == fieldName then\n\t\t\t\t\t found = true\n\t\t\t\t\t break\n\t\t\t\t end\n\t\t\t end\n\t\t\t if not found then\n\t\t\t\t local keyStr = type(k) == \"string\" and k or tostring(k)\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, keyStr)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t return \"{\" .. table.concat(items, \",\") .. \"}\"\n\t end\n else\n\t return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n\t return value\n end\n local copy = {}\n for k, v in pairs(value) do\n\t if not (ignoreKeys and ignoreKeys[k]) then\n\t\t copy[k] = deepCopy(v, ignoreKeys)\n\t end\n end\n return copy\nend\n\nlocal GRAPH_FIELD_ORDER = {\n message = {\n \"id\", \"azureTenantId\", \"azureSubscriptionId\", \"riskScore\", \"tags\", \"activityGroupName\", \"assignedTo\",\n \"category\", \"closedDateTime\", \"comments\", \"confidence\", \"createdDateTime\", \"description\", \"detectionIds\",\n \"eventDateTime\", \"feedback\", \"incidentIds\", \"lastEventDateTime\", \"lastModifiedDateTime\", \"recommendedActions\",\n \"severity\", \"sourceMaterials\", \"status\", \"title\", \"vendorInformation\", \"alertDetections\",\n \"cloudAppStates\", \"fileStates\", \"hostStates\", \"historyStates\", \"investigationSecurityStates\",\n \"malwareStates\", \"messageSecurityStates\", \"networkConnections\", \"processes\", \"registryKeyStates\", \"securityResources\",\n \"triggers\", \"userStates\", \"uriClickSecurityStates\", \"vulnerabilityStates\"\n },\n vendorInformation = {\n \"provider\", \"providerVersion\", \"subProvider\", \"vendor\", \n },\n userStates = {\n \"aadUserId\", \"accountName\", \"domainName\", \"emailRole\", \"isVpn\", \"logonDateTime\", \n \"logonId\", \"logonIp\", \"logonLocation\", \"logonType\", \"onPremisesSecurityIdentifier\",\n \"riskScore\", \"userAccountType\", \"userPrincipalName\"\n }\n}\n\nlocal MGMT_FIELD_ORDER = {\n [\"dlprulematch\"] = {\n message = {\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\",\n \"RecordType\",\n \"UserKey\",\n \"UserType\",\n \"Version\",\n \"Workload\",\n \"ObjectId\",\n \"UserId\",\n \"IncidentId\",\n \"PolicyDetails\",\n \"SensitiveInfoDetectionIsIncluded\",\n \"SharePointMetaData\",\n },\n PolicyDetails = {\n \"PolicyId\", \"PolicyName\", \"Rules\"\n },\n Rules = {\n \"ActionParameters\", \"Actions\", \"ConditionsMatched\", \"ManagementRuleId\", \"RuleId\", \"RuleMode\", \"RuleName\", \"Severity\"\n },\n ConditionsMatched = {\n \"ConditionMatchedInNewScheme\", \"OtherConditions\", \"SensitiveInformation\"\n },\n OtherConditions = {\n \"Name\", \"Value\"\n },\n SensitiveInformation = {\n \"ClassifierType\", \"Confidence\", \"Count\", \"SensitiveInformationDetailedClassificationAttributes\", \n \"SensitiveInformationDetections\", \"SensitiveInformationTypeName\", \"SensitiveType\"\n },\n SensitiveInformationDetailedClassificationAttributes = {\n \"Confidence\", \"Count\", \"IsMatch\"\n },\n SensitiveInformationDetections = {\n \"DetectedValues\", \"ResultsTruncated\"\n },\n DetectedValues = {\n \"Name\", \"Value\"\n },\n SharePointMetaData = {\n \"FileID\", \"FileName\", \"FileOwner\", \"FilePathUrl\", \"FileSize\", \"From\", \"IsViewableByExternalUsers\",\n \"IsVisibleOnlyToOdbOwner\", \"ItemCreationTime\", \"ItemLastModifiedTime\", \"ItemLastSharedTime\",\n \"SensitivityLabelIds\", \"SharedBy\", \"SiteAdmin\", \"SiteCollectionGuid\", \"SiteCollectionUrl\", \"UniqueID\"\n }\n },\n [\"dlpruleundo\"] = {\n message = {\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\",\n \"RecordType\",\n \"UserKey\",\n \"UserType\",\n \"Version\",\n \"Workload\",\n \"ObjectId\",\n \"UserId\",\n \"IncidentId\",\n \"PolicyDetails\",\n \"SensitiveInfoDetectionIsIncluded\",\n \"SharePointMetaData\",\n },\n ExceptionInfo = {\n \"Reason\"\n },\n PolicyDetails = {\n \"PolicyId\", \"PolicyName\", \"Rules\"\n },\n Rules = {\n \"ConditionsMatched\", \"ManagementRuleId\", \"OverriddenActions\", \"RuleId\", \"RuleMode\", \"RuleName\", \"Severity\"\n },\n ConditionsMatched = {\n \"ConditionMatchedInNewScheme\", \"SensitiveInformation\"\n },\n SensitiveInformation = {\n \"ClassifierType\", \"Confidence\", \"Count\", \"SensitiveInformationDetailedClassificationAttributes\", \n \"SensitiveInformationDetections\", \"SensitiveInformationTypeName\", \"SensitiveType\"\n },\n SensitiveInformationDetailedClassificationAttributes = {\n \"Confidence\", \"Count\", \"IsMatch\"\n },\n SensitiveInformationDetections = {\n \"DetectedValues\", \"ResultsTruncated\"\n },\n DetectedValues = {\n \"Name\", \"Value\"\n },\n SharePointMetaData = {\n \"FileID\", \"FileName\", \"FileOwner\", \"FilePathUrl\", \"FileSize\", \"From\", \"IsViewableByExternalUsers\",\n \"IsVisibleOnlyToOdbOwner\", \"ItemCreationTime\", \"ItemLastModifiedTime\", \"ItemLastSharedTime\",\n \"SensitivityLabelIds\", \"SharedBy\", \"SiteAdmin\", \"SiteCollectionGuid\", \"SiteCollectionUrl\", \"UniqueID\"\n }\n \n },\n [\"update user\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"userloggedin\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ClientIP\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"ActorIpAddress\", \"InterSystemsId\", \n \"IntraSystemId\", \"SupportTicketId\", \"Target\", \"TargetContextId\", \"ApplicationId\", \"DeviceProperties\", \"ErrorNumber\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n },\n DeviceProperties = {\n \"Name\", \"Value\"\n }\n },\n [\"add group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \"ExtendedProperties\",\n \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\", \"SupportTicketId\", \n \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"add member to group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"delete group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"update group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"remove member from group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"add user\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"reset user password\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"delete user\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"addedtogroup\"] = {\n message = {\n \"AppAccessContext\",\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\", \n \"RecordType\", \n \"UserKey\",\n \"UserType\", \n \"Version\", \n \"Workload\", \n \"ClientIP\", \n \"ObjectId\", \n \"UserId\", \n \"CorrelationId\", \n \"EventSource\", \n \"ItemType\", \n \"Site\", \n \"UserAgent\", \n \"WebId\", \n \"EventData\", \n \"TargetUserOrGroupType\", \n \"SiteUrl\", \n \"TargetUserOrGroupName\"\n },\n AppAccessContext = {\n \"AADSessionId\", \"CorrelationId\", \"UniqueTokenId\"\n },\n },\n [\"signinevent\"] = {\n message = {\n \"AppAccessContext\",\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\", \n \"RecordType\", \n \"UserKey\",\n \"UserType\", \n \"Version\", \n \"Workload\", \n \"ClientIP\", \n \"UserId\", \n \"AuthenticationType\",\n \"BrowserName\",\n \"BrowserVersion\",\n \"CorrelationId\",\n \"EventSource\",\n \"IsManagedDevice\",\n \"ItemType\",\n \"Platform\",\n \"UserAgent\",\n \"DeviceDisplayName\"\n },\n AppAccessContext = {\n \"AADSessionId\", \"CorrelationId\", \"UniqueTokenId\"\n },\n },\n}\n\nlocal function get_field_order(api, event_type)\n if api == \"Graph\" then\n return GRAPH_FIELD_ORDER or {}\n else\n local et = (event_type or \"\"):lower()\n return MGMT_FIELD_ORDER[et] or {}\n end\nend\n\nlocal IGNORE_KEYS = {\n _ob = true,\n timestamp = true, -- Ignore timestamp as we use start_time/end_time\n}\n\n-- Global entry point\nfunction processEvent(event)\n local e = event or {}\n -- If input already has OCSF-shaped dotted keys (post-mapped), just build nested and return\n if e[\"metadata.original_time\"] or e[\"category_uid\"] or e[\"class_uid\"] or e[\"dataSource.vendor\"] then\n return build_nested(e)\n end\n -- allow site id from event.site.id or event.site_id\n local site_id = nil\n local st = e[\"site\"]\n if type(st) == \"table\" and st[\"id\"] then site_id = st[\"id\"] elseif e[\"site_id\"] then site_id = e[\"site_id\"] end\n\n local api = detect_api(e)\n local working = cloneTable(e)\n\n if api == \"Graph\" then\n working = set_graph_synthetic_fields(working, site_id)\n local result = default_graph_api_parser(working)\n result.time = convertUtcToMilliseconds(e[\"createdDateTime\"])\n -- Store original input as JSON string in message field (with field ordering)\n local originalInput = deepCopy(event, IGNORE_KEYS) or {}\n result.message = encodeJson(originalInput, get_field_order(api, \"\"), \"message\")\n return result\n else\n working = set_mgmt_synthetic_fields(working, site_id)\n local result = default_mgmt_api_parser(working)\n result.time = convertUtcToMilliseconds(e[\"CreationTime\"])\n -- Store original input as JSON string in message field (with field ordering)\n local originalInput = deepCopy(event, IGNORE_KEYS) or {}\n local event_type = tostring(working[\"Operation\"] or \"Other\"):gsub(\"%.\", \"\")\n result.message = encodeJson(originalInput, get_field_order(api, event_type), \"message\")\n return result\n end\nend\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/microsoft_365/sample.json b/pipelines/community/transform_ocsf/microsoft_365/sample.json deleted file mode 100644 index defada4..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/sample.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "id": "cb1628d2-7c07-4435-9f4e-87fd44024d05", - "azureSubscriptionId": "6deb1e8b-896d-484a-a649-aeb6aebfeaf9", - "azureTenantId": "8c7e1716-01d8-4aeb-ac89-c691e8957e6f", - "activityGroupName": "Suspicious email forwarding", - "assignedTo": "unassigned", - "category": "MaliciousEmail", - "closedDateTime": "2026-04-20T13:40:52.634172Z", - "cloudAppStates": [ - { - "destinationServiceIp": "135.249.79.95", - "destinationServiceName": "SharePoint Online", - "riskScore": "66" - } - ], - "comments": [ - "Alert generated at 2026-04-20 03:33:52", - "Escalated to security team" - ], - "confidence": 97, - "createdDateTime": "2026-04-20T03:33:52.634172Z", - "description": "Microsoft 365 security alert: Suspicious email forwarding. This alert indicates potential maliciousemail activity detected in your environment.", - "detectionIds": [ - "de01dafc-efd2-47a5-9d71-9ca354ae2bc5" - ], - "eventDateTime": "2026-04-20T03:36:52.634172Z", - "feedback": "falsePositive", - "incidentIds": [], - "lastEventDateTime": "2026-04-20T03:54:52.634172Z", - "lastModifiedDateTime": "2026-04-20T03:40:52.634172Z", - "malwareStates": [], - "networkConnections": [ - { - "applicationName": "chrome.exe", - "destinationAddress": "199.227.89.52", - "destinationDomain": "graph.microsoft.com", - "destinationPort": "8080", - "destinationUrl": "https://domain-62.com/api/data", - "direction": "Outbound", - "domainRegisteredDateTime": "2025-07-18T03:40:52.634250Z", - "localDnsName": "workstation-937.internal.com", - "natDestinationAddress": "54.175.90.160", - "natDestinationPort": "41509", - "natSourceAddress": "10.79.101.231", - "natSourcePort": "40428", - "protocol": "UDP", - "riskScore": "24", - "sourceAddress": "10.246.169.54", - "sourcePort": "56074", - "status": "Timeout", - "urlCategory": "Malicious" - }, - { - "applicationName": "teams.exe", - "destinationAddress": "77.78.37.51", - "destinationDomain": "suspicious-domain.com", - "destinationPort": "993", - "destinationUrl": "https://suspicious-domain-9.com/api/data", - "direction": "Inbound", - "domainRegisteredDateTime": "2025-12-29T03:40:52.634286Z", - "localDnsName": "workstation-611.company.com", - "natDestinationAddress": "112.48.136.205", - "natDestinationPort": "13031", - "natSourceAddress": "10.81.153.186", - "natSourcePort": "62464", - "protocol": "ICMP", - "riskScore": "70", - "sourceAddress": "10.106.169.99", - "sourcePort": "24233", - "status": "Timeout", - "urlCategory": "Malicious" - }, - { - "applicationName": "outlook.exe", - "destinationAddress": "143.207.0.118", - "destinationDomain": "outlook.office365.com", - "destinationPort": "443", - "destinationUrl": "https://suspicious-domain-61.com/api/data", - "direction": "Outbound", - "domainRegisteredDateTime": "2026-04-02T03:40:52.634305Z", - "localDnsName": "workstation-181.internal.com", - "natDestinationAddress": "69.11.88.152", - "natDestinationPort": "30303", - "natSourceAddress": "10.117.172.211", - "natSourcePort": "16650", - "protocol": "ICMP", - "riskScore": "82", - "sourceAddress": "10.221.94.206", - "sourcePort": "37257", - "status": "Timeout", - "urlCategory": "Technology" - } - ], - "processes": [ - { - "accountName": "bob.jones", - "commandLine": "C:\\Windows\\System32\\regsvr32.exe -ExecutionPolicy Bypass 1c17ec96df224690", - "createdDateTime": "2026-04-20T03:15:52.634324Z", - "fileHash": { - "hashType": "sha256", - "hashValue": "ca810c03ed064d33ad983d1f289575f907a64d598cb04f709955f5ac1e617080" - }, - "integrityLevel": "Low", - "isElevated": "true", - "name": "regsvr32.exe", - "parentProcessCreatedDateTime": "2026-04-20T03:21:52.634333Z", - "parentProcessId": 8701, - "parentProcessName": "explorer.exe", - "path": "C:\\Windows\\System32\\regsvr32.exe", - "processId": 5212 - } - ], - "recommendedActions": [ - "Investigate user activity", - "Review mailbox rules", - "Check for additional indicators", - "Validate with user" - ], - "registryKeyStates": [], - "securityResources": [ - "Microsoft Cloud App Security", - "Azure Security Center", - "Microsoft Defender for Office 365" - ], - "severity": "critical", - "sourceMaterials": [ - "https://security.microsoft.com/alerts/f76fd6f1-7179-4045-a0ad-ca79d7ac572d", - "https://portal.office.com/adminportal/home#/MessageCenter/:/messages/cd845bfa-9e65-4051-896b-0b44d35b0cdd" - ], - "status": "dismissed", - "tags": [ - "maliciousemail", - "m365", - "security", - "automated" - ], - "title": "Microsoft 365 Alert: Suspicious email forwarding", - "triggers": [ - { - "name": "MaliciousEmail_detection_rule", - "type": "SecurityEvent", - "value": "79" - } - ], - "userStates": [ - { - "aadUserId": "5bdb7c45-546d-4f19-8b7c-b24815554c57", - "accountName": "jane.smith", - "domainName": "company.com", - "emailRole": "sender", - "isVpn": "true", - "logonDateTime": "2026-04-19T07:40:52.634379Z", - "logonId": "903143", - "logonIp": "89.36.216.146", - "logonLocation": "Seattle, WA", - "logonType": "Batch", - "onPremisesSecurityIdentifier": "S-1-5-21-797898851-168380601-9172-4847", - "riskScore": "29", - "userAgent": "Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.14326; Pro)", - "userPrincipalName": "jane.smith@company.com" - } - ], - "vendorInformation": { - "provider": "Microsoft", - "providerVersion": "1.0", - "subProvider": "Microsoft 365 Defender", - "vendor": "Microsoft" - }, - "riskScore": "75" -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/microsoft_365/serializer.lua b/pipelines/community/transform_ocsf/microsoft_365/serializer.lua deleted file mode 100644 index f4bfeda..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/serializer.lua +++ /dev/null @@ -1,1342 +0,0 @@ --- Microsoft 365 (O365) OCSF 1.0.0 parser (ported from Python Parsers/microsoft) - --- Helpers - -local function convertUtcToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - -local function split(str, delimiter) - local result = {} - local s = tostring(str or "") - local escaped = delimiter:gsub("[%.%+%*%?%^%$%(%)%[%]%%]", "%%%1") - local pattern = "([^" .. escaped .. "]+)" - for token in s:gmatch(pattern) do table.insert(result, token) end - if #result == 0 and #s > 0 then table.insert(result, s) end - return result -end - -local function getByPath(obj, keys) - local cur = obj - for _, k in ipairs(keys) do - if cur ~= nil and type(cur) == "table" then cur = cur[k] else return nil end - end - return cur -end - -local function setByDottedPath(t, dotted, value) - local keys = split(dotted, ".") - local cur = t - for i=1,#keys-1 do - local k = keys[i] - if type(cur[k]) ~= "table" then cur[k] = {} end - cur = cur[k] - end - cur[keys[#keys]] = value -end - -local function flatten_table(tbl, prefix, out) - out = out or {} - prefix = prefix or "" - - for k, v in pairs(tbl) do - local key - - -- ARRAY HANDLING (0-based index) - if type(k) == "number" then - key = string.format("%s[%d]", prefix, k - 1) - - -- NORMAL OBJECT KEY - else - if prefix ~= "" then - key = prefix .. "." .. k - else - key = k - end - end - - -- recurse - if type(v) == "table" then - flatten_table(v, key, out) - else - out[key] = v - end - end - - return out -end - -local function apply_mapping(event, mapping) - local out = {} - - -- normal mapping (existing behaviour) - for src, dst in pairs(mapping) do - local val = getByPath(event, split(src, ".")) - if val ~= nil then out[dst] = val end - end - - -- NEW: auto-capture unmapped fields - local flat_event = flatten_table(event) - for src, val in pairs(flat_event) do - if mapping[src] == nil then - out["unmapped." .. src] = val - end - end - - return out -end - -local function build_nested(flat) - local nested = {} - for k, v in pairs(flat) do setByDottedPath(nested, k, v) end - return nested -end - -local function cloneTable(src) - local dst = {} - for k,v in pairs(src or {}) do dst[k] = v end - return dst -end - --- Simple email detection -local function is_email(s) - if type(s) ~= "string" then return false end - return s:find("@") ~= nil -end - --- Common + synthetic field helpers -local function get_status_default_ocsf_mapping(status) - if status == "Success" then return 1 - elseif status == "Failure" then return 2 - else return 99 end -end - -local function get_device_os_type(platform) - local mapping = { Win = 100, Android = 201, ["iOS"] = 301 } - if type(platform) ~= "string" then return 99 end - return mapping[platform] or 99 -end - -local function get_actor_type_details(user_type) - local map = { - [0] = { type = "User", id = 1 }, - [2] = { type = "Admin", id = 2 }, - [4] = { type = "System", id = 4 }, - } - local rec = map[user_type] - if rec then return rec.type, rec.id else return "Other", 99 end -end - -local function get_state_id(event_type) - if type(event_type) == "string" and event_type:lower() == "dlprulematch" then return 1 else return 0 end -end - -local function extract_policy_details(log) - local analytic_id, analytic_name = nil, nil - local pd = log["PolicyDetails"] - if type(pd) == "table" and #pd > 0 and type(pd[1]) == "table" then - if pd[1]["PolicyId"] then analytic_id = pd[1]["PolicyId"] end - if pd[1]["PolicyName"] then analytic_name = pd[1]["PolicyName"] end - end - return analytic_id, analytic_name -end - -local function get_device_property(log) - local device_props = log["DeviceProperties"] - if type(device_props) == "table" then - for _, dp in ipairs(device_props) do - if type(dp) == "table" then - if dp["Name"] == "OS" then - log["device_os_type"] = dp["Value"] - log["device_os_type_id"] = (dp["Value"] == "Linux" and 200) or (dp["Value"] == "Windows" and 100) or 99 - elseif dp["Name"] == "SessionId" then - log["actor_session_uid"] = dp["Value"] - elseif dp["Name"] == "IsCompliantAndManaged" then - log["device_is_compliant"] = dp["Value"] - log["device_is_managed"] = dp["Value"] - end - end - end - end - return log -end - -local function get_resource_data(log) - local out = {} - local mprops = log["ModifiedProperties"] - if type(mprops) == "table" then - for _, obj in ipairs(mprops) do - if type(obj) == "table" and obj["Name"] then - out[obj["Name"]] = { NewValue = obj["NewValue"], OldValue = obj["OldValue"] } - end - end - end - return out -end - -local function get_mgmt_observables(log) - local obs = {} - local sp = log["SharePointMetaData"] - if type(sp) == "table" then - if sp["FileName"] then table.insert(obs, { type_id = 7, type = "FileName", name = "process.file.name", value = sp["FileName"] }) end - if sp["From"] then table.insert(obs, { type_id = 4, type = "User Name", name = "unmapped.sharePointMetaData.from", value = sp["From"] }) end - end - return obs -end - --- Synthetic for Graph API -local function set_graph_synthetic_fields(log, site_id) - log["category_uid"] = 2 - log["class_uid"] = 2001 - log["class_name"] = "Security Finding" - log["type_uid"] = 200199 - log["type_name"] = "Security Finding: Other" - log["OCSF_version"] = "1.0.0" - log["category_name"] = "Findings" - log["activity_id"] = 99 - log["dataSource"] = { name = "Microsoft O365", category = "security", vendor = "Microsoft" } - log["event"] = { type = log["status"] } - log["cloud"] = { provider = "Microsoft Azure" } - if site_id then log["site"] = { id = site_id } end - -- observables - local observables = {} - if log["azureTenantId"] then table.insert(observables, { type_id = 10, type = "Resource UID", name = "unmapped.azureTenantId", value = log["azureTenantId"] }) end - log["observables"] = observables - if type(log["sourceMaterials"]) == "table" then log["sourceMaterials"] = log["sourceMaterials"][1] end - return log -end - --- Severity for Mgmt -local function get_severity_id(event_type) - local m = { - dlprulematch = 1, dlpruleundo = 0, ["update user"] = 0, userloggedin = 0, ["add group"] = 0, - ["add member to group"] = 0, ["delete group"] = 0, ["update group"] = 0, ["remove member from group"] = 0, - ["add user"] = 0, ["reset user password"] = 0, ["delete user"] = 0, addedtogroup = 0, signinevent = 0, - } - return m[(event_type or ""):lower()] or 0 -end - -local function get_mgmt_log_mapping_fields(event_type) - local map = { - ["dlprulematch"] = { - activity = { name = "Create", id = 99 }, class = { name = "Security Finding", id = 2001 }, category = { name = "Findings", id = 2 }, - }, - ["dlpruleundo"] = { - activity = { name = "DLPRuleUndo", id = 99 }, class = { name = "Security Finding", id = 2001 }, category = { name = "Findings", id = 2 }, - }, - ["update user"] = { - activity = { name = "Update user", id = 99 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["userloggedin"] = { - activity = { name = "Logon", id = 1 }, class = { name = "Authentication", id = 3002 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["add group"] = { - activity = { name = "Add group", id = 99 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["add member to group"] = { - activity = { name = "Add User", id = 3 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["delete group"] = { - activity = { name = "Delete group", id = 99 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["update group"] = { - activity = { name = "Update group.", id = 99 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["remove member from group"] = { - activity = { name = "Remove User", id = 4 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["add user"] = { - activity = { name = "Create", id = 1 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["reset user password"] = { - activity = { name = "Password Reset", id = 4 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["delete user"] = { - activity = { name = "Delete", id = 6 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["addedtogroup"] = { - activity = { name = "Add User", id = 3 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["signinevent"] = { - activity = { name = "Logon", id = 1 }, class = { name = "Authentication", id = 3002 }, category = { name = "Identity & Access Management", id = 3 }, - }, - } - local m = map[(event_type or ""):lower()] or { activity = { name = "Other", id = 99 }, class = { name = "Base Event", id = 0}, category = { name = "Uncategorized", id = 0 } } - local class_name, class_id = m.class.name, m.class.id - local activity_name, activity_id = m.activity.name, m.activity.id - local type_name, type_id = string.format("%s: %s", class_name, activity_name), (class_id * 100) + activity_id - local category_name, category_id = m.category.name, m.category.id - return activity_name, activity_id, class_name, class_id, type_name, type_id, category_name, category_id -end - --- Mapping tables from python -local function get_default_graph_ocsf_mapping() - return { - ["activityGroupName"] = "resources.group_name", - ["category"] = "finding.types", - ["closedDateTime"] = "end_time_dt", - ["confidence"] = "confidence", - ["createdDateTime"] = "metadata.original_time", - ["eventDateTime"] = "start_time_dt", - ["id"] = "metadata.uid", - ["lastModifiedDateTime"] = "finding.modified_time", - ["sourceMaterials"] = "finding.src_url", - ["status"] = "activity_name", - ["vendorInformation.provider"] = "metadata.product.name", - ["vendorInformation.providerVersion"] = "metadata.product.version", - ["vendorInformation.vendor"] = "metadata.product.vendor_name", - ["lastEventDateTime"] = "end_time", - ["title"] = "finding.title", - ["category_uid"] = "category_uid", - ["class_uid"] = "class_uid", - ["type_uid"] = "type_uid", - ["category_name"] = "category_name", - ["class_name"] = "class_name", - ["type_name"] = "type_name", - ["activity_id"] = "activity_id", - ["cloud.provider"] = "cloud.provider", - ["OCSF_version"] = "metadata.version", - ["observables"] = "observables", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["event.type"] = "event.type", - ["dataSource.name"] = "dataSource.name", - ["dataSource.vendor"] = "dataSource.vendor", - } -end - -local function generic_mgmt_mapping() - return { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - } -end - -local function get_mgmt_default_mapping(event_type) - -- ported from python OCSFMapping.get_mgmt_default_mapping - local default = { - ["dlprulematch"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "finding.title", - ["OrganizationId"] = "cloud.org.uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["IncidentId"] = "finding.uid", - ["SharePointMetaData.FileID"] = "process.file.uid", - ["SharePointMetaData.FileName"] = "process.file.name", - ["SharePointMetaData.FileOwner"] = "process.file.owner.uid", - ["SharePointMetaData.FilePathUrl"] = "process.file.path", - ["SharePointMetaData.FileSize"] = "process.file.size", - ["SharePointMetaData.From"] = "resources.data.from", - ["SharePointMetaData.IsViewableByExternalUsers"] = "resources.data.is_viewable_by_external_users", - ["SharePointMetaData.IsVisibleOnlyToOdbOwner"] = "resources.data.is_visible_only_to_Odb_owner", - ["SharePointMetaData.ItemCreationTime"] = "finding.created_time", - ["SharePointMetaData.ItemLastModifiedTime"] = "finding.modified_time", - ["SharePointMetaData.ItemLastSharedTime"] = "resources.data.item_last_shared_time", - ["SharePointMetaData.SensitivityLabelIds"] = "resources.data.sensitivity_label_ids", - ["SharePointMetaData.SharedBy"] = "resources.data.shared_by", - ["SharePointMetaData.SiteAdmin"] = "resources.data.site_admin", - ["SharePointMetaData.SiteCollectionGuid"] = "resources.data.site_collection_guid", - ["SharePointMetaData.SiteCollectionUrl"] = "resources.data.site_collection_url", - ["SharePointMetaData.UniqueID"] = "resources.data.unique_id", - ["state_id"] = "state_id", - ["status_id"] = "status_id", - ["analytic.uid"] = "analytic.uid", - ["analytic.name"] = "analytic.name", - ["analytic_type_id"] = "analytic.type_id", - ["process_file_owner_type_id"] = "process.file.owner.type_id", - }, - ["dlpruleundo"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "finding.title", - ["OrganizationId"] = "cloud.org.uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["IncidentId"] = "finding.uid", - ["SharePointMetaData.FileID"] = "process.file.uid", - ["SharePointMetaData.FileName"] = "process.file.name", - ["SharePointMetaData.FileOwner"] = "process.file.owner.uid", - ["SharePointMetaData.FilePathUrl"] = "process.file.path", - ["SharePointMetaData.FileSize"] = "process.file.size", - ["SharePointMetaData.From"] = "resources.data.from", - ["SharePointMetaData.IsViewableByExternalUsers"] = "resources.data.is_viewable_by_external_users", - ["SharePointMetaData.IsVisibleOnlyToOdbOwner"] = "resources.data.is_visible_only_to_Odb_owner", - ["SharePointMetaData.ItemCreationTime"] = "finding.created_time", - ["SharePointMetaData.ItemLastModifiedTime"] = "finding.modified_time", - ["SharePointMetaData.ItemLastSharedTime"] = "resources.data.item_last_shared_time", - ["SharePointMetaData.SensitivityLabelIds"] = "resources.data.sensitivity_label_ids", - ["SharePointMetaData.SharedBy"] = "resources.data.shared_by", - ["SharePointMetaData.SiteAdmin"] = "resources.data.site_admin", - ["SharePointMetaData.SiteCollectionGuid"] = "resources.data.site_collection_guid", - ["SharePointMetaData.SiteCollectionUrl"] = "resources.data.site_collection_url", - ["SharePointMetaData.UniqueID"] = "resources.data.unique_id", - ["analytic.uid"] = "analytic.uid", - ["analytic.name"] = "analytic.name", - ["analytic_type_id"] = "analytic.type_id", - ["process_file_owner_type_id"] = "process.file.owner.type_id", - ["state_id"] = "state_id", - }, - ["update user"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "activity_name", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["userloggedin"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ClientIP"] = "device.ip", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["ActorIpAddress"] = "src_endpoint.ip", - ["InterSystemsId"] = "metadata.correlation_uid", - ["ApplicationId"] = "dst_endpoint.uid", - ["ErrorNumber"] = "api.response.code", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["device_os_type"] = "device.os.type", - ["device_os_type_id"] = "device.os.type_id", - ["actor_session_uid"] = "actor.session.uid", - ["device_is_compliant"] = "device.is_compliant", - ["device_is_managed"] = "device.is_managed", - }, - ["add group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "activity_name", - ["OrganizationId"] = "cloud.org.uid", - ["ResultStatus"] = "status_detail", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["add member to group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["ResultStatus"] = "status_detail", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ActorContextId"] = "actor.user.org.uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["delete group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["update group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "activity_name", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["remove member from group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["add user"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["reset user password"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["delete user"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.UserKey", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["addedtogroup"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ClientIP"] = "device.ip", - ["CorrelationId"] = "metadata.correlation_uid", - ["EventSource"] = "resource.name", - ["EventData"] = "status_detail", - ["TargetUserOrGroupType"] = "user.groups.type", - ["TargetUserOrGroupName"] = "user.groups.name", - ["AppAccessContext.CorrelationId"] = "actor.idp.uid", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - }, - ["signinevent"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ClientIP"] = "src_endpoint.ip", - ["CorrelationId"] = "metadata.correlation_uid", - ["EventSource"] = "service.name", - ["IsManagedDevice"] = "device.is_managed", - ["Platform"] = "device.os.type", - ["UserAgent"] = "http_request.user_agent", - ["DeviceDisplayName"] = "device.ip", - ["AppAccessContext.AADSessionId"] = "actor.session.uid", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["device_os_type_id"] = "device.os.type_id", - }, - } - local et = (event_type or ""):lower() - return default[et] or generic_mgmt_mapping() -end - -local function common_mapping() - return { - ["category_name"] = "category_name", - ["category_uid"] = "category_uid", - ["class_uid"] = "class_uid", - ["severity_id"] = "severity_id", - ["activity_name"] = "activity_name", - ["activity_id"] = "activity_id", - ["type_uid"] = "type_uid", - ["product_vendor_name"] = "metadata.product.vendor_name", - ["product_name"] = "metadata.product.name", - ["OCSF_version"] = "metadata.version", - ["observables"] = "observables", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["event.type"] = "event.type", - ["dataSource.name"] = "dataSource.name", - ["dataSource.vendor"] = "dataSource.vendor", - ["message"] = "message", - ["class_name"] = "class_name", - ["type_name"] = "type_name", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["status_id"] = "status_id", - } -end - --- Mgmt synthetic enrichment -local function set_mgmt_synthetic_fields(log, site_id) - log["product_vendor_name"] = "Microsoft" - log["OCSF_version"] = "1.0.0" - local event_type = tostring(log["Operation"] or "Other") - local event_type_dup = event_type:gsub("%.", "") - - log["severity_id"] = get_severity_id(event_type_dup) - local activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid = get_mgmt_log_mapping_fields(event_type_dup) - log["activity_name"], log["activity_id"], log["class_name"], log["class_uid"], log["type_name"], log["type_uid"], log["category_name"], log["category_uid"] = activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid - - log["status_id"] = get_status_default_ocsf_mapping(log["ResultStatus"]) - - if site_id then log["site"] = { id = site_id } end - - if log["activity_id"] == 99 then - log["event"] = { type = event_type } - log["activity_name"] = event_type - else - log["event"] = { type = log["activity_name"] } - end - - log["dataSource"] = { name = "Microsoft O365", category = "security", vendor = "Microsoft" } - - if event_type_dup:lower() == "dlprulematch" or event_type_dup:lower() == "dlpruleundo" then - log["analytic_type_id"] = 1 - log["process_file_owner_type_id"] = 99 - log["observables"] = get_mgmt_observables(log) - log["state_id"] = get_state_id(event_type_dup) - local aid, aname = extract_policy_details(log) - log["analytic"] = { uid = aid, name = aname } - else - -- set actor values - if (event_type_dup:lower() == "userloggedin") then - log["actor_user_type"], log["actor_user_type_id"] = "User", 1 - end - local userId = log["UserId"] - if is_email(userId) then - log["actor_user_email_addr"] = userId - elseif type(userId) == "string" and userId ~= "" then - log["actor_user_uid"] = userId - end - local actors = log["Actor"] - if type(actors) == "table" and #actors > 0 then - for _, a in ipairs(actors) do - if type(a) == "table" then - local id = a["ID"] - if is_email(id) then - log["actor_user_email_addr"] = id - local t, tid = get_actor_type_details(a["Type"]) - log["actor_user_type"], log["actor_user_type_id"] = t, tid - break - elseif type(id) == "string" and id:match("-") then - log["actor_user_uid"] = id - local t, tid = get_actor_type_details(a["Type"]) - log["actor_user_type"], log["actor_user_type_id"] = t, tid - break - end - end - end - else - local t, tid = get_actor_type_details(log["UserType"]) - log["actor_user_type"], log["actor_user_type_id"] = t, tid - end - - -- set user target data - local target = log["Target"] - if type(target) == "table" and #target > 0 and type(target[1]) == "table" then - local tid = target[1]["ID"] - if is_email(tid) then log["user_email_addr"] = tid else log["user_uid"] = tid end - local t, tidv = get_actor_type_details(target[1]["Type"]) - log["user_type"], log["user_type_id"] = t, tidv - end - end - - if log["UserKey"] == "Not Available" or log["UserKey"] == "NA" then log["UserKey"] = "" end - - local etl = event_type_dup:lower() - if etl == "add group" or etl == "add member to group" or etl == "delete group" or etl == "update group" or etl == "remove member from group" then - log["resource"] = { data = get_resource_data(log) } - end - - if etl == "signinevent" then - log["device_os_type_id"] = get_device_os_type(log["Platform"]) - end - - if etl == "userloggedin" then - log = get_device_property(log) - end - - return log -end - --- Parsers -local function default_graph_api_parser(alertLog) - return build_nested(apply_mapping(alertLog, get_default_graph_ocsf_mapping())) -end - -local function default_mgmt_api_parser(log) - local event_type = tostring(log["Operation"] or "Other"):gsub("%.", "") - local mapping = get_mgmt_default_mapping(event_type) - -- merge with common mapping - for src, dst in pairs(common_mapping()) do mapping[src] = dst end - local flat = apply_mapping(log, mapping) - return build_nested(flat) -end - --- API detection: Graph has createdDateTime, vendorInformation; Mgmt has CreationTime/Operation -local function detect_api(event) - if event["createdDateTime"] or getByPath(event, {"vendorInformation"}) then return "Graph" end - return "Mgmt" -end - --- Helper: Encode Lua table to JSON string with field ordering -local function encodeJson(obj, fieldOrder, key) - if obj == nil then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray then - local items = {} - for i = 1, maxIndex do - table.insert(items, encodeJson(obj[i], fieldOrder, key)) - end - return "[" .. table.concat(items, ",") .. "]" - else - local items = {} - local fieldOrdering = fieldOrder[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrdering) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldOrder, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - -- Phase 2: remaining keys (not in fieldOrder) - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrdering) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, fieldOrder, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - return "{" .. table.concat(items, ",") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - -local GRAPH_FIELD_ORDER = { - message = { - "id", "azureTenantId", "azureSubscriptionId", "riskScore", "tags", "activityGroupName", "assignedTo", - "category", "closedDateTime", "comments", "confidence", "createdDateTime", "description", "detectionIds", - "eventDateTime", "feedback", "incidentIds", "lastEventDateTime", "lastModifiedDateTime", "recommendedActions", - "severity", "sourceMaterials", "status", "title", "vendorInformation", "alertDetections", - "cloudAppStates", "fileStates", "hostStates", "historyStates", "investigationSecurityStates", - "malwareStates", "messageSecurityStates", "networkConnections", "processes", "registryKeyStates", "securityResources", - "triggers", "userStates", "uriClickSecurityStates", "vulnerabilityStates" - }, - vendorInformation = { - "provider", "providerVersion", "subProvider", "vendor", - }, - userStates = { - "aadUserId", "accountName", "domainName", "emailRole", "isVpn", "logonDateTime", - "logonId", "logonIp", "logonLocation", "logonType", "onPremisesSecurityIdentifier", - "riskScore", "userAccountType", "userPrincipalName" - } -} - -local MGMT_FIELD_ORDER = { - ["dlprulematch"] = { - message = { - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ObjectId", - "UserId", - "IncidentId", - "PolicyDetails", - "SensitiveInfoDetectionIsIncluded", - "SharePointMetaData", - }, - PolicyDetails = { - "PolicyId", "PolicyName", "Rules" - }, - Rules = { - "ActionParameters", "Actions", "ConditionsMatched", "ManagementRuleId", "RuleId", "RuleMode", "RuleName", "Severity" - }, - ConditionsMatched = { - "ConditionMatchedInNewScheme", "OtherConditions", "SensitiveInformation" - }, - OtherConditions = { - "Name", "Value" - }, - SensitiveInformation = { - "ClassifierType", "Confidence", "Count", "SensitiveInformationDetailedClassificationAttributes", - "SensitiveInformationDetections", "SensitiveInformationTypeName", "SensitiveType" - }, - SensitiveInformationDetailedClassificationAttributes = { - "Confidence", "Count", "IsMatch" - }, - SensitiveInformationDetections = { - "DetectedValues", "ResultsTruncated" - }, - DetectedValues = { - "Name", "Value" - }, - SharePointMetaData = { - "FileID", "FileName", "FileOwner", "FilePathUrl", "FileSize", "From", "IsViewableByExternalUsers", - "IsVisibleOnlyToOdbOwner", "ItemCreationTime", "ItemLastModifiedTime", "ItemLastSharedTime", - "SensitivityLabelIds", "SharedBy", "SiteAdmin", "SiteCollectionGuid", "SiteCollectionUrl", "UniqueID" - } - }, - ["dlpruleundo"] = { - message = { - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ObjectId", - "UserId", - "IncidentId", - "PolicyDetails", - "SensitiveInfoDetectionIsIncluded", - "SharePointMetaData", - }, - ExceptionInfo = { - "Reason" - }, - PolicyDetails = { - "PolicyId", "PolicyName", "Rules" - }, - Rules = { - "ConditionsMatched", "ManagementRuleId", "OverriddenActions", "RuleId", "RuleMode", "RuleName", "Severity" - }, - ConditionsMatched = { - "ConditionMatchedInNewScheme", "SensitiveInformation" - }, - SensitiveInformation = { - "ClassifierType", "Confidence", "Count", "SensitiveInformationDetailedClassificationAttributes", - "SensitiveInformationDetections", "SensitiveInformationTypeName", "SensitiveType" - }, - SensitiveInformationDetailedClassificationAttributes = { - "Confidence", "Count", "IsMatch" - }, - SensitiveInformationDetections = { - "DetectedValues", "ResultsTruncated" - }, - DetectedValues = { - "Name", "Value" - }, - SharePointMetaData = { - "FileID", "FileName", "FileOwner", "FilePathUrl", "FileSize", "From", "IsViewableByExternalUsers", - "IsVisibleOnlyToOdbOwner", "ItemCreationTime", "ItemLastModifiedTime", "ItemLastSharedTime", - "SensitivityLabelIds", "SharedBy", "SiteAdmin", "SiteCollectionGuid", "SiteCollectionUrl", "UniqueID" - } - - }, - ["update user"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["userloggedin"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ClientIP", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "ActorIpAddress", "InterSystemsId", - "IntraSystemId", "SupportTicketId", "Target", "TargetContextId", "ApplicationId", "DeviceProperties", "ErrorNumber" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - }, - DeviceProperties = { - "Name", "Value" - } - }, - ["add group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", "ExtendedProperties", - "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", "SupportTicketId", - "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["add member to group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["delete group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["update group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["remove member from group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["add user"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["reset user password"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["delete user"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["addedtogroup"] = { - message = { - "AppAccessContext", - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ClientIP", - "ObjectId", - "UserId", - "CorrelationId", - "EventSource", - "ItemType", - "Site", - "UserAgent", - "WebId", - "EventData", - "TargetUserOrGroupType", - "SiteUrl", - "TargetUserOrGroupName" - }, - AppAccessContext = { - "AADSessionId", "CorrelationId", "UniqueTokenId" - }, - }, - ["signinevent"] = { - message = { - "AppAccessContext", - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ClientIP", - "UserId", - "AuthenticationType", - "BrowserName", - "BrowserVersion", - "CorrelationId", - "EventSource", - "IsManagedDevice", - "ItemType", - "Platform", - "UserAgent", - "DeviceDisplayName" - }, - AppAccessContext = { - "AADSessionId", "CorrelationId", "UniqueTokenId" - }, - }, -} - -local function get_field_order(api, event_type) - if api == "Graph" then - return GRAPH_FIELD_ORDER or {} - else - local et = (event_type or ""):lower() - return MGMT_FIELD_ORDER[et] or {} - end -end - -local IGNORE_KEYS = { - _ob = true, - timestamp = true, -- Ignore timestamp as we use start_time/end_time -} - --- Global entry point -function processEvent(event) - local e = event or {} - -- If input already has OCSF-shaped dotted keys (post-mapped), just build nested and return - if e["metadata.original_time"] or e["category_uid"] or e["class_uid"] or e["dataSource.vendor"] then - return build_nested(e) - end - -- allow site id from event.site.id or event.site_id - local site_id = nil - local st = e["site"] - if type(st) == "table" and st["id"] then site_id = st["id"] elseif e["site_id"] then site_id = e["site_id"] end - - local api = detect_api(e) - local working = cloneTable(e) - - if api == "Graph" then - working = set_graph_synthetic_fields(working, site_id) - local result = default_graph_api_parser(working) - result.time = convertUtcToMilliseconds(e["createdDateTime"]) - -- Store original input as JSON string in message field (with field ordering) - local originalInput = deepCopy(event, IGNORE_KEYS) or {} - result.message = encodeJson(originalInput, get_field_order(api, ""), "message") - return result - else - working = set_mgmt_synthetic_fields(working, site_id) - local result = default_mgmt_api_parser(working) - result.time = convertUtcToMilliseconds(e["CreationTime"]) - -- Store original input as JSON string in message field (with field ordering) - local originalInput = deepCopy(event, IGNORE_KEYS) or {} - local event_type = tostring(working["Operation"] or "Other"):gsub("%.", "") - result.message = encodeJson(originalInput, get_field_order(api, event_type), "message") - return result - end -end diff --git a/pipelines/community/transform_ocsf/okta/metadata.yaml b/pipelines/community/transform_ocsf/okta/metadata.yaml deleted file mode 100644 index 617525c..0000000 --- a/pipelines/community/transform_ocsf/okta/metadata.yaml +++ /dev/null @@ -1,53 +0,0 @@ -grade: - letter: F - score: 45 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Okta. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: okta - dataSource: Okta - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "API Key & Secret" - sample_record: "{\n \"actor\": {\n \"id\": \"00u4729hjsVRU197Y5d7\",\n \"type\": \"User\",\n\ - \ \"alternateId\": \"pnnpydhb@example.com\",\n \"displayName\": \"Pnnpydhb\",\n \"detailEntry\"\ - : null\n },\n \"client\": {\n \"userAgent\": {\n \"rawUserAgent\": \"Mozilla/5.0 (X11; Linux\ - \ x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36\",\n \"os\":\ - \ \"Linux\",\n \"browser\": \"CHROME\"\n },\n \"zone\": \"null\",\n \"device\": \"Computer\"\ - ,\n \"id\": null,\n \"ipAddress\": \"95.196.9.145\",\n \"geographicalContext\": {\n \ - \ \"city\": \"Hyderabad\",\n \"state\": \"Telangana\",\n \"country\": \"India\",\n \ - \ \"postalCode\": \"500004\",\n \"geolocation\": {\n \"lat\": 17.3724,\n \"lon\"\ - : 78.4378\n }\n }\n },\n \"device\": null,\n \"authenticationContext\": {\n \"authenticationProvider\"\ - : null,\n \"credentialProvider\": null,\n \"credentialType\": null,\n \"issuer\": null,\n\ - \ \"interface\": null,\n \"authenticationStep\": 0,\n \"rootSessionId\": \"102eA1R\",\n \ - \ \"externalSessionId\": \"102eA1R2M\"\n },\n \"displayMessage\": \"Create API token\",\n \"\ - eventType\": \"system.api_token.create\",\n \"outcome\": {\n \"result\": \"SUCCESS\",\n \"\ - reason\": null\n },\n \"published\": \"2026-04-20T02:26:37.000Z\",\n \"securityContext\": {\n \ - \ \"asNumber\": 150008,\n \"asOrg\": \"s r fibernet\",\n \"isp\": \"pioneer elabs ltd.\",\n\ - \ \"domain\": null,\n \"isProxy\": false\n },\n \"severity\": \"OTHER\",\n \"debugContext\"\ - : {\n \"debugData\": {\n \"concurrencyPercentage\": \"50\",\n \"requestId\": \"923c54e8-a3b4-4518-bc64-00b228adbe84\"\ - ," - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: okta, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: F - harness_score: 45 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/okta/okta.json b/pipelines/community/transform_ocsf/okta/okta.json deleted file mode 100644 index 571b587..0000000 --- a/pipelines/community/transform_ocsf/okta/okta.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Okta", - "grade": { - "letter": "F", - "score": 45, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Okta. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "okta", - "source_name": "okta", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "okta-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "\n-- Lua implementation for Okta OCSF 1.0.0 schema\n\n-- Helper function to safely access nested dictionary keys\nfunction safelyAccessNestedDictKeys(keys, dictObject)\n local current = dictObject\n for _, key in ipairs(keys) do\n if current and type(current) == \"table\" then\n current = current[key]\n else\n return nil\n end\n end\n return current\nend\n\n-- Helper function to split string by delimiter\nfunction split(str, delimiter)\n local result = {}\n -- Escape special regex characters in delimiter\n local escapedDelimiter = delimiter:gsub(\"[%.%+%*%?%^%$%(%)%[%]%%]\", \"%%%1\")\n local pattern = \"([^\" .. escapedDelimiter .. \"]+)\"\n\n for token in str:gmatch(pattern) do\n table.insert(result, token)\n end\n\n -- Handle empty string case\n if #result == 0 and #str > 0 then\n table.insert(result, str)\n end\n\n return result\nend\n\n-- Helper function to convert timestamp to milliseconds\nfunction convertToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n \n -- Parse ISO 8601 timestamp (e.g., \"2023-04-24T04:55:30.535Z\")\n local year, month, day, hour, min, sec, ms = timestamp:match(\"(%d%d%d%d)-(%d%d)-(%d%d)T(%d%d):(%d%d):(%d%d)%.(%d%d%d)Z\")\n \n if year and month and day and hour and min and sec and ms then\n -- Convert to milliseconds since epoch\n local time = os.time({\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec)\n })\n return (time * 1000) + tonumber(ms)\n end\n \n return nil\nend\n\n-- Ordered JSON encoding helpers (FIELD_ORDERS like Cisco Duo)\nlocal FIELD_ORDERS = {\n root = {\n \"actor\", \"client\", \"device\", \"authenticationContext\", \"displayMessage\",\n \"eventType\", \"outcome\", \"published\", \"securityContext\", \"severity\",\n \"debugContext\", \"legacyEventType\", \"transaction\", \"uuid\", \"version\",\n \"request\", \"target\"\n },\n actor = {\"id\", \"type\", \"alternateId\", \"displayName\", \"detailEntry\"},\n client = {\"userAgent\", \"zone\", \"device\", \"id\", \"ipAddress\", \"geographicalContext\"},\n userAgent = {\"rawUserAgent\", \"os\", \"browser\"},\n geographicalContext = {\"city\", \"state\", \"country\", \"postalCode\", \"geolocation\"},\n geolocation = {\"lat\", \"lon\"},\n authenticationContext = {\n \"authenticationProvider\", \"credentialProvider\", \"credentialType\", \"issuer\",\n \"interface\", \"authenticationStep\", \"externalSessionId\"\n },\n issuer = {\"id\", \"type\"},\n outcome = {\"result\", \"reason\"},\n securityContext = {\"asNumber\", \"asOrg\", \"isp\", \"domain\", \"isProxy\"},\n debugContext = {\"debugData\"},\n debugData = {\"initiationType\", \"redirectUri\", \"requestId\", \"dtHash\", \"signOnMode\", \"requestUri\", \"threatSuspected\", \"url\", \"risk\", \"deviceFingerprint\", \"authnRequestId\", \"behaviors\", \"warningPercent\", \"rateLimitBucketUuid\", \"rateLimitSecondsToReset\", \"threshold\", \"timeSpan\", \"rateLimitScopeType\", \"userId\", \"timeUnit\"},\n transaction = {\"type\", \"id\", \"detail\"},\n request = {\"ipChain\"},\n ipChain = {\"ip\", \"geographicalContext\", \"version\", \"source\"},\n dst_endpoint = {\"uid\", \"svc_name\"},\n user = {\"id\", \"email_addr\", \"name\"},\n target = {\"id\", \"type\", \"alternateId\", \"displayName\", \"detailEntry\"}\n}\n\nlocal function encodeWithFieldOrder(obj, fieldOrder)\n local items = {}\n -- Phase 1: predefined order\n for _, fieldName in ipairs(fieldOrder) do\n if obj[fieldName] ~= nil then\n local valueStr = encodeJson(obj[fieldName], fieldName)\n if valueStr ~= nil then\n table.insert(items, '\"' .. fieldName .. '\": ' .. valueStr)\n end\n end\n end\n -- Phase 2: remaining fields\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then found = true; break end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local valueStr = encodeJson(v, k)\n if valueStr ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. valueStr)\n end\n end\n end\n -- If no items were added, return nil to skip empty objects\n if #items == 0 then\n return nil\n end\n return \"{\" .. table.concat(items, \", \") .. \"}\"\nend\n\n-- JSON encoding function (from AWS CloudTrail) - Compact single-line output\nfunction encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" or obj == \"\" then\n return nil -- Skip null and empty fields entirely\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n\n if isArray then\n local items = {}\n for i = 1, maxIndex do\n local elementKey = key or tostring(i)\n local encoded = encodeJson(obj[i], elementKey)\n if encoded ~= nil then\n table.insert(items, encoded)\n end\n end\n -- Return nil for empty arrays to skip them entirely\n if #items == 0 then\n return nil\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n -- Check if this is an empty object - if so, return nil to skip it\n if next(obj) == nil then\n return nil\n end\n\n -- If we have a field order for this object key, use it\n if key and FIELD_ORDERS[key] then\n return encodeWithFieldOrder(obj, FIELD_ORDERS[key])\n end\n -- If top-level message root, use root order\n if key == \"root\" and FIELD_ORDERS.root then\n return encodeWithFieldOrder(obj, FIELD_ORDERS.root)\n end\n -- Fallback: unordered (natural) encoding\n local items = {}\n for k, v in pairs(obj) do\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local valueStr = encodeJson(v, keyStr)\n -- Only include fields that have non-nil values\n if valueStr ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. valueStr)\n end\n end\n -- If no items were added, return nil to skip empty objects\n if #items == 0 then\n return nil\n end\n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\n-- Get default mapping for event type\nfunction getDefaultMapping(eventType)\n local defaultMapping = {\n [\"user.session.start\"] = {\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"version\", target = \"api.version\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"authenticationContext.externalSessionId\", target = \"session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.authenticationProvider\", target = \"service.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.origin\", target = \"device.hostname\"},\n {source = \"debugContext.debugData.risk\", target = \"device.risk_level\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"}\n },\n [\"user.authentication.sso\"] = {\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"authenticationContext.externalSessionId\", target = \"session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.authenticationProvider\", target = \"service.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.signOnMode\", target = \"auth_protocol\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"},\n -- target fields\n {source = \"dst_endpoint_uid\", target = \"dst_endpoint.uid\"},\n {source = \"actor_user_uid\", target = \"actor.user.uid\"},\n {source = \"dst_endpoint_svc_name\", target = \"dst_endpoint.svc_name\"},\n {source = \"actor_email_addr\", target = \"actor.email_addr\"}\n --{source = \"actor_user_name\", target = \"user.name\"}\n },\n [\"user.lifecycle.activate\"] = {\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.browser\", target = \"client.userAgent.browser\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"}\n -- target\n --{source = \"user_id\", target = \"user.id\"}\n --{source = \"user_email_addr\", target = \"user.email_addr\"}\n --{source = \"user_name\", target = \"user.name\"}\n },\n [\"user.lifecycle.create\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestId\", target = \"api.request.uid\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"}\n -- target\n --{source = \"user_id\", target = \"user.id\"}\n --{source = \"user_email_addr\", target = \"user.email_addr\"},\n --{source = \"user_name\", target = \"user.name\"}\n },\n [\"user.lifecycle.deactivate\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestId\", target = \"api.request.uid\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"}\n -- target\n --{source = \"user_id\", target = \"user.id\"},\n --{source = \"user_email_addr\", target = \"user.email_addr\"},\n --{source = \"user_name\", target = \"user.name\"}\n },\n [\"policy.evaluate_sign_on\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"authenticationContext.authenticationProvider\", target = \"name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"authenticationContext.externalSessionId\", target = \"session.uid\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.deviceFingerprint\", target = \"device.uid\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.risk\", target = \"device.risk_level\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"},\n -- target fields\n {source = \"actor.authorization.policy.uid\", target = \"actor.authorization.policy.uid\"},\n {source = \"actor.authorization.policy.name\", target = \"actor.authorization.policy.name\"},\n {source = \"actor.authorization.policy.rule.uid\", target = \"actor.authorization.policy.rule.uid\"},\n {source = \"actor.authorization.policy.rule.name\", target = \"actor.authorization.policy.rule.name\"}\n },\n [\"system.org.rate_limit.warning\"] = {\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"version\", target = \"api.version\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"}\n },\n [\"application.user_membership.add\"] = {\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"version\", target = \"api.version\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.origin\", target = \"device.hostname\"},\n {source = \"debugContext.debugData.risk\", target = \"device.risk_level\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n -- target fields\n {source = \"dst_endpoint_uid\", target = \"dst_endpoint.uid\"},\n {source = \"dst_endpoint_svc_name\", target = \"dst_endpoint.svc_name\"},\n {source = \"actor_user_uid\", target = \"actor.user.uid\"},\n {source = \"actor_email_addr\", target = \"actor.email_addr\"}\n --{source = \"actor_user_name\", target = \"user.name\"}\n },\n [\"application.lifecycle.update\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n -- target fields\n {source = \"dst_endpoint_uid\", target = \"dst_endpoint.uid\"},\n {source = \"dst_endpoint_svc_name\", target = \"dst_endpoint.svc_name\"},\n\n --{source = \"actor_user_uid\", target = \"actor.user.uid\"},\n {source = \"actor_email_addr\", target = \"actor.email_addr\"}\n --{source = \"actor_user_name\", target = \"user.name\"}\n }\n }\n\n return defaultMapping[eventType] or getGenericOktaMapping()\nend\n\n-- Generic Okta mapping\nfunction getGenericOktaMapping()\n return {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"}\n }\nend\n\n-- Common mapping\nfunction getCommonMapping()\n return {\n {source = \"category_name\", target = \"category_name\"},\n {source = \"category_uid\", target = \"category_uid\"},\n {source = \"class_uid\", target = \"class_uid\"},\n {source = \"severity_id\", target = \"severity_id\"},\n {source = \"activity_name\", target = \"activity_name\"},\n {source = \"activity_id\", target = \"activity_id\"},\n {source = \"type_uid\", target = \"type_uid\"},\n {source = \"vendor_name\", target = \"metadata.product.vendor_name\"},\n {source = \"name\", target = \"metadata.product.name\"},\n {source = \"OCSF_version\", target = \"metadata.version\"},\n {source = \"time\", target = \"time\"},\n {source = \"status_id\", target = \"status_id\"},\n {source = \"type_id\", target = \"actor.user.type_id\"},\n {source = \"observables\", target = \"observables\"},\n {source = \"dataSource.category\", target = \"dataSource.category\"},\n {source = \"site.id\", target = \"site.id\"},\n {source = \"dataSource.name\", target = \"dataSource.name\"},\n {source = \"dataSource.vendor\", target = \"dataSource.vendor\"},\n {source = \"message\", target = \"message\"},\n {source = \"class_name\", target = \"class_name\"},\n {source = \"type_name\", target = \"type_name\"}\n }\nend\n\n-- Parse security domain\nfunction parseSecurityDomain(parsedLog)\n if parsedLog[\"src_endpoint.location.domain\"] == \".\" then\n parsedLog[\"src_endpoint.location.domain\"] = nil\n end\n return parsedLog\nend\n\n-- Set site ID\nfunction setSiteId(oktaLog, siteId)\n oktaLog[\"site\"] = {id = siteId}\n return oktaLog\nend\n\n-- Parse risk level\nfunction parseRiskLevel(parsedLog)\n local riskLevelMapper = {\n [\"Info\"] = 0, [\"Low\"] = 1, [\"Medium\"] = 2,\n [\"High\"] = 3, [\"Critical\"] = 4\n }\n local riskPrefix = \"level=\"\n local finalRiskLevel = \"Other\"\n local finalRiskLevelId = 99\n\n -- Look for risk level in the already-mapped device.risk_level field\n local riskLevel = \"\"\n if parsedLog[\"device\"] and parsedLog[\"device\"][\"risk_level\"] then\n riskLevel = parsedLog[\"device\"][\"risk_level\"]\n end\n \n for riskLevelName, riskLevelId in pairs(riskLevelMapper) do\n if string.find(string.lower(riskLevel), string.lower(riskPrefix .. riskLevelName)) then\n finalRiskLevel = riskLevelName\n finalRiskLevelId = riskLevelId\n break\n end\n end\n\n -- Set the parsed risk level back to the device object\n if not parsedLog[\"device\"] then parsedLog[\"device\"] = {} end\n parsedLog[\"device\"][\"risk_level\"] = finalRiskLevel\n parsedLog[\"device\"][\"risk_level_id\"] = finalRiskLevelId\n return parsedLog\nend\n\n-- Find event type\nfunction findEventType(log)\n local eventType = log[\"eventType\"]\n if eventType == nil then\n return \"unknown\"\n end\n return eventType\nend\n\n-- Get category UID\nfunction getCategoryUID(eventType)\n local categoryMapping = {\n [\"user.session.start\"] = 3,\n [\"user.authentication.sso\"] = 3,\n [\"user.lifecycle.activate\"] = 2,\n [\"user.lifecycle.create\"] = 3,\n [\"user.lifecycle.deactivate\"] = 3,\n [\"policy.evaluate_sign_on\"] = 3,\n [\"system.org.rate_limit.warning\"] = 3,\n [\"application.user_membership.add\"] = 3,\n [\"application.lifecycle.update\"] = 6\n }\n return categoryMapping[eventType] or 0\nend\n\n-- Get class mapping\nfunction getClassMapping(eventType)\n local classMapping = {\n [\"user.session.start\"] = {name = \"Authentication\", id = 3002},\n [\"user.authentication.sso\"] = {name = \"Authentication\", id = 3002},\n [\"user.lifecycle.activate\"] = {name = \"Account Change\", id = 3001},\n [\"user.lifecycle.create\"] = {name = \"Account Change\", id = 3001},\n [\"user.lifecycle.deactivate\"] = {name = \"Account Change\", id = 3001},\n [\"policy.evaluate_sign_on\"] = {name = \"Authentication\", id = 3002},\n [\"system.org.rate_limit.warning\"] = {name = \"API Activity\", id = 6003},\n [\"application.user_membership.add\"] = {name = \"Account Change\", id = 3001},\n [\"application.lifecycle.update\"] = {name = \"Application Lifecycle\", id = 6002}\n }\n local mapping = classMapping[eventType] or {name = \"Base Event\", id = 0}\n return mapping.name, mapping.id\nend\n\n-- Get status default OCSF mapping\nfunction getStatusDefaultOCSFMapping(status)\n if status == nil then\n return 0\n end\n\n local statusMapping = {\n [\"SUCCESS\"] = 1,\n [\"FAILURE\"] = 2\n }\n return statusMapping[status] or 99\nend\n\n-- Get activity name\nfunction getActivityName(eventType, eventTypeDisplayName)\n local activityMapping = {\n [\"user.session.start\"] = {name = \"Logon\", id = 1},\n [\"user.authentication.sso\"] = {name = \"Logon\", id = 1},\n [\"user.lifecycle.activate\"] = {name = \"Enable\", id = 2},\n [\"user.lifecycle.create\"] = {name = \"Create\", id = 1},\n [\"user.lifecycle.deactivate\"] = {name = \"Disable\", id = 5},\n [\"policy.evaluate_sign_on\"] = {name = \"Logon\", id = 1},\n [\"application.user_membership.add\"] = {name = \"Attach Policy\", id = 7}\n }\n local mapping = activityMapping[eventType] or {name = eventTypeDisplayName, id = 99}\n return mapping.name, mapping.id\nend\n\n-- Get user type\nfunction getUserType(oktaUserType)\n local userTypeMapping = {\n [\"Unknown\"] = 0,\n [\"User\"] = 1,\n [\"Admin\"] = 2,\n [\"System\"] = 3,\n [\"Other\"] = 99\n }\n for user, id in pairs(userTypeMapping) do\n if string.find(oktaUserType or \"\", user) then\n return id\n end\n end\n return 99\nend\n\n-- Get category mapper\nfunction getCategoryMapper(eventType)\n local categoryMapper = {\n [\"user.session.start\"] = \"Identity & Access Management\",\n [\"user.authentication.sso\"] = \"Identity & Access Management\",\n [\"user.lifecycle.activate\"] = \"Identity & Access Management\",\n [\"user.lifecycle.create\"] = \"Identity & Access Management\",\n [\"user.lifecycle.deactivate\"] = \"Identity & Access Management\",\n [\"policy.evaluate_sign_on\"] = \"Identity & Access Management\",\n [\"system.org.rate_limit.warning\"] = \"Application Activity\",\n [\"application.user_membership.add\"] = \"Identity & Access Management\",\n [\"application.lifecycle.update\"] = \"Application Activity\"\n }\n return categoryMapper[eventType] or \"Uncategorized\"\nend\n\n-- Get type name\nfunction getTypeName(eventType)\n local typeMapper = {\n [\"user.session.start\"] = \"Authentication: Logon\",\n [\"user.authentication.sso\"] = \"Authentication: Logon\",\n [\"user.lifecycle.activate\"] = \"Account Change: Enable\",\n [\"user.lifecycle.create\"] = \"Account Change: Create\",\n [\"user.lifecycle.deactivate\"] = \"Account Change: Disable\",\n [\"policy.evaluate_sign_on\"] = \"Authentication: Logon\",\n [\"system.org.rate_limit.warning\"] = \"API Activity: Other\",\n [\"application.user_membership.add\"] = \"Account Change: Attach Policy\",\n [\"application.lifecycle.update\"] = \"Application Lifecycle: Other\"\n }\n return typeMapper[eventType] or \"Base Event: Other\"\nend\n\n-- Get observables\nfunction getObservables(log)\n local observables = {}\n\n -- Hostname observable\n local hostname = safelyAccessNestedDictKeys({\"debugContext\", \"debugData\", \"origin\"}, log)\n if hostname and hostname ~= \"\" and hostname ~= \"null\" then\n table.insert(observables, {\n type_id = 1,\n type = \"Hostname\",\n name = \"device.hostname\",\n value = hostname\n })\n end\n\n -- IP Address observable\n local clientIpAddress = safelyAccessNestedDictKeys({\"client\", \"ipAddress\"}, log)\n if clientIpAddress and clientIpAddress ~= \"\" and clientIpAddress ~= \"null\" then\n table.insert(observables, {\n type_id = 2,\n type = \"IP Address\",\n name = \"src_endpoint.ip\",\n value = clientIpAddress\n })\n end\n\n -- User Name observable\n local userName = safelyAccessNestedDictKeys({\"actor\", \"displayName\"}, log)\n if userName and userName ~= \"\" and userName ~= \"null\" then\n table.insert(observables, {\n type_id = 4,\n type = \"User Name\",\n name = \"actor.user.name\",\n value = userName\n })\n end\n\n -- Email Address observable\n local emailAddress = safelyAccessNestedDictKeys({\"actor\", \"alternateId\"}, log)\n if emailAddress and emailAddress ~= \"\" and emailAddress ~= \"null\" then\n table.insert(observables, {\n type_id = 5,\n type = \"Email Address\",\n name = \"actor.user.email_addr\",\n value = emailAddress\n })\n end\n\n -- URL String observable\n local requestUri = safelyAccessNestedDictKeys({\"debugContext\", \"debugData\", \"requestUri\"}, log)\n if requestUri and requestUri ~= \"\" and requestUri ~= \"null\" then\n table.insert(observables, {\n type_id = 6,\n type = \"URL String\",\n name = \"http_request.url.path\",\n value = requestUri\n })\n end\n\n -- Geo Location observable\n local lat = safelyAccessNestedDictKeys({\"client\", \"geographicalContext\", \"geolocation\", \"lat\"}, log)\n local lon = safelyAccessNestedDictKeys({\"client\", \"geographicalContext\", \"geolocation\", \"lon\"}, log)\n local notAllowedItems = {{}, {}, \"\", nil}\n local latAllowed = true\n local lonAllowed = true\n for _, item in ipairs(notAllowedItems) do\n if lat == item then latAllowed = false end\n if lon == item then lonAllowed = false end\n end\n if latAllowed and lonAllowed and lat and lon then\n table.insert(observables, {\n type_id = 26,\n type = \"Geo Location\",\n name = \"client.geographicalContext.geolocation\",\n value = lat .. \", \" .. lon\n })\n end\n\n return observables\nend\n\n-- Process target fields into flat keys for mapping\nfunction processTargetFields(log, eventType)\n local targetFields = {}\n \n if eventType == \"user.authentication.sso\" or\n eventType == \"application.user_membership.add\" or\n eventType == \"application.lifecycle.update\" then\n local target = log[\"target\"]\n if target and type(target) == \"table\" then\n for _, t in ipairs(target) do\n if type(t) == \"table\" then\n if t[\"type\"] == \"AppInstance\" then\n log[\"dst_endpoint_uid\"] = t[\"type\"]\n log[\"dst_endpoint_svc_name\"] = t[\"displayName\"]\n elseif t[\"type\"] == \"AppUser\" then\n -- Override specific fields with target values\n log[\"actor_user_uid\"] = t[\"type\"] -- \"AppUser\" overrides actor.id\n log[\"actor_email_addr\"] = t[\"alternateId\"] -- \"unknown\" for actor.email_addr\n log[\"actor_user_name\"] = t[\"displayName\"] -- \"John Cena\" (same as original)\n \n -- Override actor.id for actor.user.uid mapping\n if log[\"actor\"] then\n log[\"actor\"][\"id\"] = t[\"type\"] -- \"AppUser\" for actor.user.uid\n -- Keep actor.alternateId as original for actor.user.email_addr mapping\n -- Don't override actor.alternateId - let it stay as \"john.cena@wwe.com\"\n end\n end\n end\n end\n end\n elseif eventType == \"user.lifecycle.activate\" or\n eventType == \"user.lifecycle.create\" or\n eventType == \"user.lifecycle.deactivate\" then\n local target = log[\"target\"]\n if target and type(target) == \"table\" and #target > 0 then\n local t = target[1]\n if type(t) == \"table\" then\n targetFields[\"user_id\"] = t[\"id\"]\n targetFields[\"user_email_addr\"] = t[\"alternateId\"]\n targetFields[\"user_name\"] = t[\"displayName\"]\n end\n end\n elseif eventType == \"policy.evaluate_sign_on\" then\n local target = log[\"target\"]\n if target and type(target) == \"table\" then\n for _, t in ipairs(target) do\n if type(t) == \"table\" then\n if t[\"type\"] == \"PolicyEntity\" then\n targetFields[\"actor\"] = {\n authorization = {\n policy = {\n name = t[\"displayName\"],\n uid = t[\"id\"]\n }\n }\n }\n elseif t[\"type\"] == \"PolicyRule\" then\n targetFields[\"actor\"] = {\n authorization = {\n policy = {\n rule = {\n uid = t[\"id\"],\n name = t[\"displayName\"]\n }\n }\n }\n }\n end\n end\n end\n end\n end\n\n return targetFields\nend\n\n-- Generate severity mapping\nfunction generateSeverityMapping(availableSeverityList)\n local defaultSeverityMapping = {\n [\"DEBUG\"] = 0, [\"INFO\"] = 1, [\"WARN\"] = 3, [\"ERROR\"] = 5, [\"OTHER\"] = 99\n }\n local defaultSeverityMappingKeys = {\"DEBUG\", \"INFO\", \"WARN\", \"ERROR\", \"OTHER\"}\n local severityIDMapping = {}\n\n for severityTypeIndex = 1, #availableSeverityList do\n if availableSeverityList[severityTypeIndex] then\n local key = defaultSeverityMappingKeys[severityTypeIndex]\n severityIDMapping[key] = defaultSeverityMapping[key]\n end\n end\n return severityIDMapping\nend\n\n-- Get severity ID\nfunction getSeverityID(eventType, logSeverity)\n local severityMapping = {\n [\"user.session.start\"] = generateSeverityMapping({true, true, true, true, true}),\n [\"user.authentication.sso\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"user.lifecycle.activate\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"user.lifecycle.create\"] = generateSeverityMapping({false, true, false, false, false}),\n [\"user.lifecycle.deactivate\"] = generateSeverityMapping({false, true, false, false, false}),\n [\"policy.evaluate_sign_on\"] = generateSeverityMapping({false, true, false, false, false}),\n [\"system.org.rate_limit.warning\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"application.user_membership.add\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"application.lifecycle.update\"] = generateSeverityMapping({false, true, false, false, false})\n }\n local eventSeverityMapping = severityMapping[eventType] or {}\n return eventSeverityMapping[logSeverity] or 99\nend\n\n-- Generate synthetic fields\nfunction generateSyntheticFields(log, eventType, originalLog)\n -- Set nested metadata fields\n if not log[\"metadata\"] then log[\"metadata\"] = {} end\n if not log[\"metadata\"][\"product\"] then log[\"metadata\"][\"product\"] = {} end\n log[\"metadata\"][\"product\"][\"vendor_name\"] = \"Okta\"\n log[\"metadata\"][\"product\"][\"name\"] = \"Okta\"\n log[\"metadata\"][\"version\"] = \"1.0.0\"\n\n local publishedTime = safelyAccessNestedDictKeys({\"published\"}, originalLog)\n if publishedTime then\n log[\"time\"] = convertToMilliseconds(publishedTime)\n end\n\n -- Use the event type passed as parameter instead of finding it again\n log[\"category_name\"] = getCategoryMapper(eventType)\n log[\"category_uid\"] = getCategoryUID(eventType)\n\n local className, classUid = getClassMapping(eventType)\n log[\"class_name\"] = className\n log[\"class_uid\"] = classUid\n\n -- Dynamic severity calculation\n log[\"severity_id\"] = getSeverityID(eventType, originalLog[\"severity\"])\n\n local activityName, activityId = getActivityName(eventType, originalLog[\"displayMessage\"] or \"Other\")\n log[\"activity_name\"] = activityName\n log[\"activity_id\"] = activityId\n\n log[\"type_uid\"] = (classUid * 100) + activityId\n log[\"type_name\"] = getTypeName(eventType)\n\n local outcomeResult = safelyAccessNestedDictKeys({\"outcome\", \"result\"}, originalLog)\n log[\"status_id\"] = getStatusDefaultOCSFMapping(outcomeResult)\n\n local actorType = safelyAccessNestedDictKeys({\"actor\", \"type\"}, originalLog)\n\n\tif not log[\"actor\"] then log[\"actor\"] = {} end\n\tif not log[\"actor\"][\"user\"] then log[\"actor\"][\"user\"] = {} end\n\tlog[\"actor\"][\"user\"][\"type_id\"] = getUserType(actorType)\n\tlog[\"actor\"][\"user\"][\"type\"] = actorType or \"User\"\n -- Actor email_addr is now set by target field mappings, don't override\n\n -- Handle postal code conversion to string\n local postalCode = safelyAccessNestedDictKeys({\"client\", \"geographicalContext\", \"postalCode\"}, log)\n if postalCode then\n if not log[\"client\"] then log[\"client\"] = {} end\n if not log[\"client\"][\"geographicalContext\"] then log[\"client\"][\"geographicalContext\"] = {} end\n log[\"client\"][\"geographicalContext\"][\"postalCode\"] = tostring(postalCode)\n end\n\n -- Target fields are now processed earlier in oktaLogsMapping\n\n\n log[\"event.type\"] = log[\"activity_name\"] or eventType\n log[\"dataSource\"] = {name = \"Okta\", category = \"security\", vendor = \"Okta\"}\n \n -- Add session field (skip root session for lifecycle events except deactivate)\n local sessionId = safelyAccessNestedDictKeys({\"authenticationContext\", \"externalSessionId\"}, originalLog)\n if not (eventType == \"user.lifecycle.activate\" or eventType == \"user.lifecycle.create\") then\n if eventType == \"user.lifecycle.deactivate\" then\n -- For deactivate events, add session to existing actor object if it exists\n if log[\"actor\"] then\n log[\"actor\"][\"session\"] = {uid = sessionId or \"unknown\"}\n else\n log[\"session\"] = {uid = sessionId or \"unknown\"}\n end\n end\n end\n \n -- Add user field from actor only for non-lifecycle events\n --local userName = safelyAccessNestedDictKeys({\"actor\", \"displayName\"}, originalLog)\n --if userName and not (eventType == \"user.lifecycle.activate\" or eventType == \"user.lifecycle.create\" or eventType == \"user.lifecycle.deactivate\" or eventType == \"user.session.start\") then\n -- log[\"user\"] = {name = userName}\n --end\n\n return log\nend\n\n\n-- Helper function to check if a field should be ignored\nfunction shouldIgnoreField(fieldName)\n\n local ignoreFields = {\n \"_okta_event_type\", \"_ob\", \"ts\", \"timestamp\"\n }\n\n for _, field in ipairs(ignoreFields) do\n if fieldName == field or string.find(fieldName, \"^\" .. field .. \"%.\") then\n return true\n end\n end\n return false\nend\n\n-- Helper function to check if a value is empty (empty object, array, or null)\nfunction isEmptyValue(value)\n if value == nil or value == \"NULL_PLACEHOLDER\" then\n return true\n end\n if type(value) == \"string\" and (value == \"\" or value == \"null\") then\n return true\n end\n if type(value) == \"table\" then\n -- Check if it's an empty table\n if next(value) == nil then\n return true\n end\n -- For unmapped fields, recursively check nested values to filter out empty nested objects\n local hasNonEmptyValues = false\n for k, v in pairs(value) do\n if not isEmptyValue(v) then\n hasNonEmptyValues = true\n break\n end\n end\n return not hasNonEmptyValues\n end\n return false\nend\n\n-- Helper function to add unmapped fields as a truly nested object (no dotted keys)\nfunction addUnmappedFields(sourceObj, targetObj, mappedFields, prefixParts)\n prefixParts = prefixParts or {}\n for key, value in pairs(sourceObj) do\n -- Skip ignored fields\n if shouldIgnoreField(key) then\n goto continue\n end\n\n -- Build current dotted path for mapped check\n local currentPathParts = {}\n for i = 1, #prefixParts do currentPathParts[i] = prefixParts[i] end\n table.insert(currentPathParts, key)\n local currentPath = table.concat(currentPathParts, \".\")\n\n -- Check if this exact path has been mapped\n local isMapped = mappedFields[currentPath] or false\n\n if not isMapped and value ~= nil then\n if type(value) == \"table\" then\n local nestedObj = {}\n addUnmappedFields(value, nestedObj, mappedFields, currentPathParts)\n \n -- Check if ALL direct children were mapped\n local allChildrenMapped = true\n for childKey, childValue in pairs(value) do\n local childPath = currentPath .. \".\" .. childKey\n if not mappedFields[childPath] then\n allChildrenMapped = false\n break\n end\n end\n \n -- Only add parent if it has unmapped children OR no children were mapped\n if next(nestedObj) and not allChildrenMapped then\n targetObj[key] = nestedObj\n end\n -- Don't add empty objects to unmapped\n else\n -- Only add non-empty values to unmapped\n if not isEmptyValue(value) then\n targetObj[key] = value\n end\n end\n end\n ::continue::\n end\nend\n\n-- Helper function to build nested object structure from flat dotted keys\nfunction buildNestedStructure(flatObj)\n local nested = {}\n for key, value in pairs(flatObj) do\n local keys = split(key, \".\")\n local current = nested\n for i = 1, #keys - 1 do\n local k = keys[i]\n if not current[k] then\n current[k] = {}\n elseif type(current[k]) ~= \"table\" then\n -- If the existing value is not a table, we can't create nested structure\n -- Skip this key to avoid conflicts\n goto continue\n end\n current = current[k]\n end\n \n -- Special handling for raw_data field - encode as JSON string\n if keys[#keys] == \"raw_data\" then\n -- For raw_data, we want the value as a JSON string, not double-encoded\n if type(value) == \"table\" then\n current[keys[#keys]] = encodeJson(value, \"raw_data\")\n else\n -- If it's already a string, use it as-is\n current[keys[#keys]] = tostring(value)\n end\n else\n current[keys[#keys]] = value\n end\n ::continue::\n end\n return nested\nend\n\n-- Helper function to set nested value (marks field as processed)\nfunction setNestedValue(obj, keys, value)\n local current = obj\n for i = 1, #keys - 1 do\n if not current[keys[i]] then\n current[keys[i]] = {}\n end\n current = current[keys[i]]\n end\n current[keys[#keys]] = value\nend\n\n\n-- Helper function to filter out ignored fields using shouldIgnoreField\nfunction filterIgnoredFields(log)\n local function filterObject(obj, prefix)\n prefix = prefix or \"\"\n local filteredObj = {}\n\n for key, value in pairs(obj) do\n local fullKey = prefix == \"\" and key or prefix .. \".\" .. key\n\n if not shouldIgnoreField(fullKey) then\n if type(value) == \"table\" then\n local filteredValue = filterObject(value, fullKey)\n if next(filteredValue) then -- Only add non-empty tables\n filteredObj[key] = filteredValue\n end\n else\n filteredObj[key] = value\n end\n end\n end\n return filteredObj\n end\n\n return filterObject(log)\nend\n\n\n-- Helper function to create ordered JSON message using FIELD_ORDER approach\nfunction createOrderedMessage(log, eventType)\n -- Filter out null values and empty strings from the log before creating message\n local filteredLog = {}\n for k, v in pairs(log) do\n if v ~= nil and v ~= \"\" and v ~= \"null\" then\n if type(v) == \"table\" then\n local filteredValue = filterNullValues(v)\n if next(filteredValue) then -- Only add non-empty tables\n filteredLog[k] = filteredValue\n end\n else\n filteredLog[k] = v\n end\n end\n end\n \n -- Apply domain filtering to the message for lifecycle events only\n if eventType == \"user.lifecycle.create\" or eventType == \"user.lifecycle.deactivate\" or eventType == \"user.session.start\" then\n filteredLog = applyDomainFiltering(filteredLog)\n end\n \n -- Use FIELD_ORDERS.root to create ordered JSON string\n local orderedJson = encodeWithFieldOrder(filteredLog, FIELD_ORDERS.root)\n return orderedJson or \"{}\"\nend\n\n-- Helper function to recursively filter out null values from nested tables\nfunction filterNullValues(obj)\n local filtered = {}\n for k, v in pairs(obj) do\n if v ~= nil and v ~= \"\" and v ~= \"null\" then\n if type(v) == \"table\" then\n local filteredValue = filterNullValues(v)\n if next(filteredValue) then -- Only add non-empty tables\n filtered[k] = filteredValue\n end\n else\n filtered[k] = v\n end\n end\n end\n return filtered\nend\n\n-- Helper function to apply domain filtering recursively\nfunction applyDomainFiltering(obj)\n if type(obj) ~= \"table\" then\n return obj\n end\n \n local filtered = {}\n for k, v in pairs(obj) do\n if type(v) == \"table\" then\n local filteredValue = applyDomainFiltering(v)\n if next(filteredValue) then -- Only add non-empty tables\n filtered[k] = filteredValue\n end\n else\n -- Skip domain fields with value \".\" and isp fields that duplicate asOrg\n if k == \"domain\" and v == \".\" then\n -- Skip this field\n elseif k == \"isp\" and obj[\"asOrg\"] and v == obj[\"asOrg\"] then\n -- Skip isp field when it's the same as asOrg\n else\n filtered[k] = v\n end\n end\n end\n return filtered\nend\n\n-- Simplified Okta logs mapping function\nfunction oktaLogsMapping(log)\n -- STEP 1: First filter out ignored fields using shouldIgnoreField\n log = filterIgnoredFields(log)\n\n -- STEP 2: Find event type first\n local eventType = findEventType(log)\n \n -- STEP 3: Create ordered JSON message from original log BEFORE any modifications\n local originalLog = {}\n for k, v in pairs(log) do\n originalLog[k] = v\n end\n log.message = createOrderedMessage(originalLog, eventType)\n\n -- STEP 4: Process target fields to create flat keys for mapping\n local targetFields = processTargetFields(log, eventType)\n \n -- Apply target fields to log for specific events so mappings can consume them\n if eventType == \"policy.evaluate_sign_on\" then\n for key, value in pairs(targetFields) do\n if key == \"actor\" then\n if not log[\"actor\"] then log[\"actor\"] = {} end\n for subKey, subValue in pairs(value) do\n log[\"actor\"][subKey] = subValue\n end\n else\n log[key] = value\n end\n end\n elseif eventType == \"user.lifecycle.activate\" or eventType == \"user.lifecycle.create\" or eventType == \"user.lifecycle.deactivate\" then\n -- For lifecycle events, expose target-derived user fields on the log\n for key, value in pairs(targetFields) do\n log[key] = value\n end\n end\n\n -- STEP 4: Get mapping for event type\n local mappings = getDefaultMapping(eventType)\n\n -- Merge with common mapping\n local commonMappings = getCommonMapping()\n for _, mapping in ipairs(commonMappings) do\n table.insert(mappings, mapping)\n end\n\n -- STEP 4: Apply mappings and track processed fields\n local parsedData = {}\n local mappedFields = {}\n for _, mapping in ipairs(mappings) do\n local sourcePath = mapping.source\n local targetPath = mapping.target\n local keys = split(sourcePath, \".\")\n local value = safelyAccessNestedDictKeys(keys, log)\n\n if value ~= nil and value ~= \"\" and value ~= \"null\" then\n if targetPath == \"src_endpoint.location.domain\" and value == \".\" then\n -- Skip this field for any event when domain is a single dot\n else\n -- Update target in parsedData\n parsedData[targetPath] = value\n -- Track mapped field by source path\n mappedFields[sourcePath] = true\n end\n end\n end\n\n -- STEP 5: Add remaining non-null fields to unmapped\n parsedData[\"unmapped\"] = {}\n local ignoreUnmappedFields = {\n \"actor_user_uid\",\n \"actor_email_addr\", \n \"actor_user_name\",\n \"user_id\",\n \"user_email_addr\",\n \"user_name\"\n }\n for _, field in ipairs(ignoreUnmappedFields) do\n mappedFields[field] = true\n end\n\n addUnmappedFields(log, parsedData[\"unmapped\"], mappedFields, {})\n \n -- Apply domain filtering to unmapped fields as well\n parsedData[\"unmapped\"] = applyDomainFiltering(parsedData[\"unmapped\"])\n\n -- STEP 6: Convert flat dotted keys to nested objects\n parsedData = buildNestedStructure(parsedData)\n\n -- STEP 7: Apply post-processing\n parsedData = parseSecurityDomain(parsedData)\n parsedData = parseRiskLevel(parsedData)\n\n -- STEP 8: Generate synthetic fields\n parsedData = generateSyntheticFields(parsedData, eventType, log)\n \n -- STEP 9: Generate observables for specific event types (after synthetic fields)\n local observablesEventTypes = {\n \"user.authentication.sso\",\n \"application.user_membership.add\",\n \"user.lifecycle.activate\",\n \"user.lifecycle.create\",\n \"user.lifecycle.deactivate\",\n \"policy.evaluate_sign_on\",\n \"system.org.rate_limit.warning\",\n \"user.session.start\"\n }\n\n for _, eventTypeName in ipairs(observablesEventTypes) do\n if eventType == eventTypeName then\n local observables = getObservables(log)\n -- Convert observables to array format\n local observablesArray = {}\n for _, obs in ipairs(observables) do\n table.insert(observablesArray, obs)\n end\n parsedData[\"observables\"] = observablesArray\n break\n end\n end\n \n parsedData = convertToNested(parsedData, eventType)\n return parsedData\nend\n\n-- Convert flat parsed data to nested JSON structure with field ordering\nfunction convertToNested(parsedData, eventType)\n local nested = {}\n \n -- First, build the nested structure\n for key, value in pairs(parsedData) do\n -- Special case: keep event.type as flattened field (don't nest it)\n if key == \"event.type\" then\n nested[\"event.type\"] = value -- Keep as flattened event.type\n goto continue\n end\n \n local keys = split(key, \".\")\n local current = nested\n \n -- Navigate to the parent object\n for i = 1, #keys - 1 do\n local k = keys[i]\n if not current[k] then\n current[k] = {}\n elseif type(current[k]) ~= \"table\" then\n -- If the existing value is not a table, we can't create nested structure\n -- Skip this key to avoid conflicts\n goto continue\n end\n current = current[k]\n end\n \n -- Set the final value - special handling for raw_data field\n if keys[#keys] == \"raw_data\" then\n -- For raw_data, we want the value as a JSON string, not double-encoded\n if type(value) == \"table\" then\n current[keys[#keys]] = encodeJson(value, \"raw_data\")\n else\n -- If it's already a string, use it as-is\n current[keys[#keys]] = tostring(value)\n end\n else\n current[keys[#keys]] = value\n end\n ::continue::\n end\n \n -- Then, apply field ordering to the nested structure\n return applyFieldOrdering(nested, nil, eventType)\nend\n\n-- Apply field ordering to nested structure\nfunction applyFieldOrdering(obj, fieldOrder, eventType)\n fieldOrder = fieldOrder or FIELD_ORDERS.root\n local ordered = {}\n \n -- Phase 1: Add fields in predefined order\n for _, fieldName in ipairs(fieldOrder) do\n if obj[fieldName] ~= nil then\n if type(obj[fieldName]) == \"table\" then\n -- Recursively apply ordering to nested objects\n local nestedFieldOrder = FIELD_ORDERS[fieldName]\n ordered[fieldName] = applyFieldOrdering(obj[fieldName], nestedFieldOrder, eventType)\n else\n ordered[fieldName] = obj[fieldName]\n end\n end\n end\n \n -- Phase 2: Add remaining fields not in the predefined order\n for key, value in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if key == fieldName then \n found = true\n break \n end\n end\n \n if not found then\n if type(value) == \"table\" then\n -- Recursively apply ordering to nested objects\n local nestedFieldOrder = FIELD_ORDERS[key]\n ordered[key] = applyFieldOrdering(value, nestedFieldOrder, eventType)\n else\n ordered[key] = value\n end\n end\n end\n \n return ordered\nend\n\n-- Convert ISO 8601 timestamp to Unix epoch milliseconds\nfunction convertToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n \n -- Parse ISO 8601 format: \"2025-09-29T09:15:40Z\" or \"2025-09-29T09:15:40.123Z\"\n local year, month, day, hour, min, sec, ms = string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z\")\n \n if year and month and day and hour and min and sec then\n local t = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n \n -- Get local time interpretation\n local local_seconds = os.time(t)\n \n -- Get what this represents in UTC\n local utc_date = os.date(\"!*t\", local_seconds)\n \n local unix_seconds\n -- Check if the UTC interpretation matches our input\n if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and \n utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and \n utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then\n -- We are already in UTC, use as-is\n unix_seconds = local_seconds\n else\n -- Calculate the correct UTC timestamp\n local utc_seconds = os.time(utc_date)\n local offset = local_seconds - utc_seconds\n unix_seconds = local_seconds + offset -- Add offset to get UTC\n end\n \n -- Add milliseconds if present\n local milli = 0\n if ms and ms ~= \"\" then\n milli = tonumber((ms .. \"000\"):sub(1, 3)) -- pad/truncate to 3 digits\n end\n \n return unix_seconds * 1000 + milli\n end\n \n return nil\nend\n\n-- Main event processing function\nfunction processEvent(event)\n return oktaLogsMapping(event)\nend", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "F", - "harness_score": 45, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/okta/sample.json b/pipelines/community/transform_ocsf/okta/sample.json deleted file mode 100644 index 31471e0..0000000 --- a/pipelines/community/transform_ocsf/okta/sample.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "actor": { - "id": "00u4729hjsVRU197Y5d7", - "type": "User", - "alternateId": "pnnpydhb@example.com", - "displayName": "Pnnpydhb", - "detailEntry": null - }, - "client": { - "userAgent": { - "rawUserAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "os": "Linux", - "browser": "CHROME" - }, - "zone": "null", - "device": "Computer", - "id": null, - "ipAddress": "95.196.9.145", - "geographicalContext": { - "city": "Hyderabad", - "state": "Telangana", - "country": "India", - "postalCode": "500004", - "geolocation": { - "lat": 17.3724, - "lon": 78.4378 - } - } - }, - "device": null, - "authenticationContext": { - "authenticationProvider": null, - "credentialProvider": null, - "credentialType": null, - "issuer": null, - "interface": null, - "authenticationStep": 0, - "rootSessionId": "102eA1R", - "externalSessionId": "102eA1R2M" - }, - "displayMessage": "Create API token", - "eventType": "system.api_token.create", - "outcome": { - "result": "SUCCESS", - "reason": null - }, - "published": "2026-04-20T02:26:37.000Z", - "securityContext": { - "asNumber": 150008, - "asOrg": "s r fibernet", - "isp": "pioneer elabs ltd.", - "domain": null, - "isProxy": false - }, - "severity": "OTHER", - "debugContext": { - "debugData": { - "concurrencyPercentage": "50", - "requestId": "923c54e8-a3b4-4518-bc64-00b228adbe84", - "dtHash": "43fa03767e62cd45bf1b9ecf03ca2ee6f8d42c1248d6c3d5adcf990c055128a0", - "rateLimitPercentage": "50", - "networkConnection": "ANYWHERE", - "requestUri": "/api/internal/tokens", - "url": "/api/internal/tokens?expand=user" - } - }, - "legacyEventType": "api.token.create", - "transaction": { - "type": "WEB", - "id": "bc6400b228adbe84923c54e8a3b44518", - "detail": {} - }, - "uuid": "b2ece00c-6728-4e85-a08b-b0e3ccdcbaf2", - "version": "0", - "request": { - "ipChain": [ - { - "ip": "103.99.111.142", - "geographicalContext": { - "city": "Hyderabad", - "state": "Telangana", - "country": "India", - "postalCode": "500004", - "geolocation": { - "lat": 17.3724, - "lon": 78.4378 - } - }, - "version": "V4", - "source": null - } - ] - }, - "target": [ - { - "id": "00T2lyn19", - "type": "Token", - "alternateId": "unknown", - "displayName": "Test", - "detailEntry": null - } - ] -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/okta/serializer.lua b/pipelines/community/transform_ocsf/okta/serializer.lua deleted file mode 100644 index c3897c8..0000000 --- a/pipelines/community/transform_ocsf/okta/serializer.lua +++ /dev/null @@ -1,1495 +0,0 @@ - --- Lua implementation for Okta OCSF 1.0.0 schema - --- Helper function to safely access nested dictionary keys -function safelyAccessNestedDictKeys(keys, dictObject) - local current = dictObject - for _, key in ipairs(keys) do - if current and type(current) == "table" then - current = current[key] - else - return nil - end - end - return current -end - --- Helper function to split string by delimiter -function split(str, delimiter) - local result = {} - -- Escape special regex characters in delimiter - local escapedDelimiter = delimiter:gsub("[%.%+%*%?%^%$%(%)%[%]%%]", "%%%1") - local pattern = "([^" .. escapedDelimiter .. "]+)" - - for token in str:gmatch(pattern) do - table.insert(result, token) - end - - -- Handle empty string case - if #result == 0 and #str > 0 then - table.insert(result, str) - end - - return result -end - --- Helper function to convert timestamp to milliseconds -function convertToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - - -- Parse ISO 8601 timestamp (e.g., "2023-04-24T04:55:30.535Z") - local year, month, day, hour, min, sec, ms = timestamp:match("(%d%d%d%d)-(%d%d)-(%d%d)T(%d%d):(%d%d):(%d%d)%.(%d%d%d)Z") - - if year and month and day and hour and min and sec and ms then - -- Convert to milliseconds since epoch - local time = os.time({ - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec) - }) - return (time * 1000) + tonumber(ms) - end - - return nil -end - --- Ordered JSON encoding helpers (FIELD_ORDERS like Cisco Duo) -local FIELD_ORDERS = { - root = { - "actor", "client", "device", "authenticationContext", "displayMessage", - "eventType", "outcome", "published", "securityContext", "severity", - "debugContext", "legacyEventType", "transaction", "uuid", "version", - "request", "target" - }, - actor = {"id", "type", "alternateId", "displayName", "detailEntry"}, - client = {"userAgent", "zone", "device", "id", "ipAddress", "geographicalContext"}, - userAgent = {"rawUserAgent", "os", "browser"}, - geographicalContext = {"city", "state", "country", "postalCode", "geolocation"}, - geolocation = {"lat", "lon"}, - authenticationContext = { - "authenticationProvider", "credentialProvider", "credentialType", "issuer", - "interface", "authenticationStep", "externalSessionId" - }, - issuer = {"id", "type"}, - outcome = {"result", "reason"}, - securityContext = {"asNumber", "asOrg", "isp", "domain", "isProxy"}, - debugContext = {"debugData"}, - debugData = {"initiationType", "redirectUri", "requestId", "dtHash", "signOnMode", "requestUri", "threatSuspected", "url", "risk", "deviceFingerprint", "authnRequestId", "behaviors", "warningPercent", "rateLimitBucketUuid", "rateLimitSecondsToReset", "threshold", "timeSpan", "rateLimitScopeType", "userId", "timeUnit"}, - transaction = {"type", "id", "detail"}, - request = {"ipChain"}, - ipChain = {"ip", "geographicalContext", "version", "source"}, - dst_endpoint = {"uid", "svc_name"}, - user = {"id", "email_addr", "name"}, - target = {"id", "type", "alternateId", "displayName", "detailEntry"} -} - -local function encodeWithFieldOrder(obj, fieldOrder) - local items = {} - -- Phase 1: predefined order - for _, fieldName in ipairs(fieldOrder) do - if obj[fieldName] ~= nil then - local valueStr = encodeJson(obj[fieldName], fieldName) - if valueStr ~= nil then - table.insert(items, '"' .. fieldName .. '": ' .. valueStr) - end - end - end - -- Phase 2: remaining fields - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then found = true; break end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local valueStr = encodeJson(v, k) - if valueStr ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. valueStr) - end - end - end - -- If no items were added, return nil to skip empty objects - if #items == 0 then - return nil - end - return "{" .. table.concat(items, ", ") .. "}" -end - --- JSON encoding function (from AWS CloudTrail) - Compact single-line output -function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" or obj == "" then - return nil -- Skip null and empty fields entirely - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray then - local items = {} - for i = 1, maxIndex do - local elementKey = key or tostring(i) - local encoded = encodeJson(obj[i], elementKey) - if encoded ~= nil then - table.insert(items, encoded) - end - end - -- Return nil for empty arrays to skip them entirely - if #items == 0 then - return nil - end - return "[" .. table.concat(items, ", ") .. "]" - else - -- Check if this is an empty object - if so, return nil to skip it - if next(obj) == nil then - return nil - end - - -- If we have a field order for this object key, use it - if key and FIELD_ORDERS[key] then - return encodeWithFieldOrder(obj, FIELD_ORDERS[key]) - end - -- If top-level message root, use root order - if key == "root" and FIELD_ORDERS.root then - return encodeWithFieldOrder(obj, FIELD_ORDERS.root) - end - -- Fallback: unordered (natural) encoding - local items = {} - for k, v in pairs(obj) do - local keyStr = type(k) == "string" and k or tostring(k) - local valueStr = encodeJson(v, keyStr) - -- Only include fields that have non-nil values - if valueStr ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. valueStr) - end - end - -- If no items were added, return nil to skip empty objects - if #items == 0 then - return nil - end - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - --- Get default mapping for event type -function getDefaultMapping(eventType) - local defaultMapping = { - ["user.session.start"] = { - {source = "uuid", target = "metadata.uid"}, - {source = "published", target = "metadata.original_time"}, - {source = "version", target = "api.version"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "authenticationContext.externalSessionId", target = "session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.authenticationProvider", target = "service.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.origin", target = "device.hostname"}, - {source = "debugContext.debugData.risk", target = "device.risk_level"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"} - }, - ["user.authentication.sso"] = { - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "authenticationContext.externalSessionId", target = "session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.authenticationProvider", target = "service.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.signOnMode", target = "auth_protocol"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"}, - -- target fields - {source = "dst_endpoint_uid", target = "dst_endpoint.uid"}, - {source = "actor_user_uid", target = "actor.user.uid"}, - {source = "dst_endpoint_svc_name", target = "dst_endpoint.svc_name"}, - {source = "actor_email_addr", target = "actor.email_addr"} - --{source = "actor_user_name", target = "user.name"} - }, - ["user.lifecycle.activate"] = { - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.browser", target = "client.userAgent.browser"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"} - -- target - --{source = "user_id", target = "user.id"} - --{source = "user_email_addr", target = "user.email_addr"} - --{source = "user_name", target = "user.name"} - }, - ["user.lifecycle.create"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestId", target = "api.request.uid"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"} - -- target - --{source = "user_id", target = "user.id"} - --{source = "user_email_addr", target = "user.email_addr"}, - --{source = "user_name", target = "user.name"} - }, - ["user.lifecycle.deactivate"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestId", target = "api.request.uid"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"} - -- target - --{source = "user_id", target = "user.id"}, - --{source = "user_email_addr", target = "user.email_addr"}, - --{source = "user_name", target = "user.name"} - }, - ["policy.evaluate_sign_on"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "authenticationContext.authenticationProvider", target = "name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "authenticationContext.externalSessionId", target = "session.uid"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.deviceFingerprint", target = "device.uid"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.risk", target = "device.risk_level"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"}, - -- target fields - {source = "actor.authorization.policy.uid", target = "actor.authorization.policy.uid"}, - {source = "actor.authorization.policy.name", target = "actor.authorization.policy.name"}, - {source = "actor.authorization.policy.rule.uid", target = "actor.authorization.policy.rule.uid"}, - {source = "actor.authorization.policy.rule.name", target = "actor.authorization.policy.rule.name"} - }, - ["system.org.rate_limit.warning"] = { - {source = "uuid", target = "metadata.uid"}, - {source = "published", target = "metadata.original_time"}, - {source = "version", target = "api.version"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"} - }, - ["application.user_membership.add"] = { - {source = "uuid", target = "metadata.uid"}, - {source = "published", target = "metadata.original_time"}, - {source = "version", target = "api.version"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.origin", target = "device.hostname"}, - {source = "debugContext.debugData.risk", target = "device.risk_level"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - -- target fields - {source = "dst_endpoint_uid", target = "dst_endpoint.uid"}, - {source = "dst_endpoint_svc_name", target = "dst_endpoint.svc_name"}, - {source = "actor_user_uid", target = "actor.user.uid"}, - {source = "actor_email_addr", target = "actor.email_addr"} - --{source = "actor_user_name", target = "user.name"} - }, - ["application.lifecycle.update"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - -- target fields - {source = "dst_endpoint_uid", target = "dst_endpoint.uid"}, - {source = "dst_endpoint_svc_name", target = "dst_endpoint.svc_name"}, - - --{source = "actor_user_uid", target = "actor.user.uid"}, - {source = "actor_email_addr", target = "actor.email_addr"} - --{source = "actor_user_name", target = "user.name"} - } - } - - return defaultMapping[eventType] or getGenericOktaMapping() -end - --- Generic Okta mapping -function getGenericOktaMapping() - return { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"} - } -end - --- Common mapping -function getCommonMapping() - return { - {source = "category_name", target = "category_name"}, - {source = "category_uid", target = "category_uid"}, - {source = "class_uid", target = "class_uid"}, - {source = "severity_id", target = "severity_id"}, - {source = "activity_name", target = "activity_name"}, - {source = "activity_id", target = "activity_id"}, - {source = "type_uid", target = "type_uid"}, - {source = "vendor_name", target = "metadata.product.vendor_name"}, - {source = "name", target = "metadata.product.name"}, - {source = "OCSF_version", target = "metadata.version"}, - {source = "time", target = "time"}, - {source = "status_id", target = "status_id"}, - {source = "type_id", target = "actor.user.type_id"}, - {source = "observables", target = "observables"}, - {source = "dataSource.category", target = "dataSource.category"}, - {source = "site.id", target = "site.id"}, - {source = "dataSource.name", target = "dataSource.name"}, - {source = "dataSource.vendor", target = "dataSource.vendor"}, - {source = "message", target = "message"}, - {source = "class_name", target = "class_name"}, - {source = "type_name", target = "type_name"} - } -end - --- Parse security domain -function parseSecurityDomain(parsedLog) - if parsedLog["src_endpoint.location.domain"] == "." then - parsedLog["src_endpoint.location.domain"] = nil - end - return parsedLog -end - --- Set site ID -function setSiteId(oktaLog, siteId) - oktaLog["site"] = {id = siteId} - return oktaLog -end - --- Parse risk level -function parseRiskLevel(parsedLog) - local riskLevelMapper = { - ["Info"] = 0, ["Low"] = 1, ["Medium"] = 2, - ["High"] = 3, ["Critical"] = 4 - } - local riskPrefix = "level=" - local finalRiskLevel = "Other" - local finalRiskLevelId = 99 - - -- Look for risk level in the already-mapped device.risk_level field - local riskLevel = "" - if parsedLog["device"] and parsedLog["device"]["risk_level"] then - riskLevel = parsedLog["device"]["risk_level"] - end - - for riskLevelName, riskLevelId in pairs(riskLevelMapper) do - if string.find(string.lower(riskLevel), string.lower(riskPrefix .. riskLevelName)) then - finalRiskLevel = riskLevelName - finalRiskLevelId = riskLevelId - break - end - end - - -- Set the parsed risk level back to the device object - if not parsedLog["device"] then parsedLog["device"] = {} end - parsedLog["device"]["risk_level"] = finalRiskLevel - parsedLog["device"]["risk_level_id"] = finalRiskLevelId - return parsedLog -end - --- Find event type -function findEventType(log) - local eventType = log["eventType"] - if eventType == nil then - return "unknown" - end - return eventType -end - --- Get category UID -function getCategoryUID(eventType) - local categoryMapping = { - ["user.session.start"] = 3, - ["user.authentication.sso"] = 3, - ["user.lifecycle.activate"] = 2, - ["user.lifecycle.create"] = 3, - ["user.lifecycle.deactivate"] = 3, - ["policy.evaluate_sign_on"] = 3, - ["system.org.rate_limit.warning"] = 3, - ["application.user_membership.add"] = 3, - ["application.lifecycle.update"] = 6 - } - return categoryMapping[eventType] or 0 -end - --- Get class mapping -function getClassMapping(eventType) - local classMapping = { - ["user.session.start"] = {name = "Authentication", id = 3002}, - ["user.authentication.sso"] = {name = "Authentication", id = 3002}, - ["user.lifecycle.activate"] = {name = "Account Change", id = 3001}, - ["user.lifecycle.create"] = {name = "Account Change", id = 3001}, - ["user.lifecycle.deactivate"] = {name = "Account Change", id = 3001}, - ["policy.evaluate_sign_on"] = {name = "Authentication", id = 3002}, - ["system.org.rate_limit.warning"] = {name = "API Activity", id = 6003}, - ["application.user_membership.add"] = {name = "Account Change", id = 3001}, - ["application.lifecycle.update"] = {name = "Application Lifecycle", id = 6002} - } - local mapping = classMapping[eventType] or {name = "Base Event", id = 0} - return mapping.name, mapping.id -end - --- Get status default OCSF mapping -function getStatusDefaultOCSFMapping(status) - if status == nil then - return 0 - end - - local statusMapping = { - ["SUCCESS"] = 1, - ["FAILURE"] = 2 - } - return statusMapping[status] or 99 -end - --- Get activity name -function getActivityName(eventType, eventTypeDisplayName) - local activityMapping = { - ["user.session.start"] = {name = "Logon", id = 1}, - ["user.authentication.sso"] = {name = "Logon", id = 1}, - ["user.lifecycle.activate"] = {name = "Enable", id = 2}, - ["user.lifecycle.create"] = {name = "Create", id = 1}, - ["user.lifecycle.deactivate"] = {name = "Disable", id = 5}, - ["policy.evaluate_sign_on"] = {name = "Logon", id = 1}, - ["application.user_membership.add"] = {name = "Attach Policy", id = 7} - } - local mapping = activityMapping[eventType] or {name = eventTypeDisplayName, id = 99} - return mapping.name, mapping.id -end - --- Get user type -function getUserType(oktaUserType) - local userTypeMapping = { - ["Unknown"] = 0, - ["User"] = 1, - ["Admin"] = 2, - ["System"] = 3, - ["Other"] = 99 - } - for user, id in pairs(userTypeMapping) do - if string.find(oktaUserType or "", user) then - return id - end - end - return 99 -end - --- Get category mapper -function getCategoryMapper(eventType) - local categoryMapper = { - ["user.session.start"] = "Identity & Access Management", - ["user.authentication.sso"] = "Identity & Access Management", - ["user.lifecycle.activate"] = "Identity & Access Management", - ["user.lifecycle.create"] = "Identity & Access Management", - ["user.lifecycle.deactivate"] = "Identity & Access Management", - ["policy.evaluate_sign_on"] = "Identity & Access Management", - ["system.org.rate_limit.warning"] = "Application Activity", - ["application.user_membership.add"] = "Identity & Access Management", - ["application.lifecycle.update"] = "Application Activity" - } - return categoryMapper[eventType] or "Uncategorized" -end - --- Get type name -function getTypeName(eventType) - local typeMapper = { - ["user.session.start"] = "Authentication: Logon", - ["user.authentication.sso"] = "Authentication: Logon", - ["user.lifecycle.activate"] = "Account Change: Enable", - ["user.lifecycle.create"] = "Account Change: Create", - ["user.lifecycle.deactivate"] = "Account Change: Disable", - ["policy.evaluate_sign_on"] = "Authentication: Logon", - ["system.org.rate_limit.warning"] = "API Activity: Other", - ["application.user_membership.add"] = "Account Change: Attach Policy", - ["application.lifecycle.update"] = "Application Lifecycle: Other" - } - return typeMapper[eventType] or "Base Event: Other" -end - --- Get observables -function getObservables(log) - local observables = {} - - -- Hostname observable - local hostname = safelyAccessNestedDictKeys({"debugContext", "debugData", "origin"}, log) - if hostname and hostname ~= "" and hostname ~= "null" then - table.insert(observables, { - type_id = 1, - type = "Hostname", - name = "device.hostname", - value = hostname - }) - end - - -- IP Address observable - local clientIpAddress = safelyAccessNestedDictKeys({"client", "ipAddress"}, log) - if clientIpAddress and clientIpAddress ~= "" and clientIpAddress ~= "null" then - table.insert(observables, { - type_id = 2, - type = "IP Address", - name = "src_endpoint.ip", - value = clientIpAddress - }) - end - - -- User Name observable - local userName = safelyAccessNestedDictKeys({"actor", "displayName"}, log) - if userName and userName ~= "" and userName ~= "null" then - table.insert(observables, { - type_id = 4, - type = "User Name", - name = "actor.user.name", - value = userName - }) - end - - -- Email Address observable - local emailAddress = safelyAccessNestedDictKeys({"actor", "alternateId"}, log) - if emailAddress and emailAddress ~= "" and emailAddress ~= "null" then - table.insert(observables, { - type_id = 5, - type = "Email Address", - name = "actor.user.email_addr", - value = emailAddress - }) - end - - -- URL String observable - local requestUri = safelyAccessNestedDictKeys({"debugContext", "debugData", "requestUri"}, log) - if requestUri and requestUri ~= "" and requestUri ~= "null" then - table.insert(observables, { - type_id = 6, - type = "URL String", - name = "http_request.url.path", - value = requestUri - }) - end - - -- Geo Location observable - local lat = safelyAccessNestedDictKeys({"client", "geographicalContext", "geolocation", "lat"}, log) - local lon = safelyAccessNestedDictKeys({"client", "geographicalContext", "geolocation", "lon"}, log) - local notAllowedItems = {{}, {}, "", nil} - local latAllowed = true - local lonAllowed = true - for _, item in ipairs(notAllowedItems) do - if lat == item then latAllowed = false end - if lon == item then lonAllowed = false end - end - if latAllowed and lonAllowed and lat and lon then - table.insert(observables, { - type_id = 26, - type = "Geo Location", - name = "client.geographicalContext.geolocation", - value = lat .. ", " .. lon - }) - end - - return observables -end - --- Process target fields into flat keys for mapping -function processTargetFields(log, eventType) - local targetFields = {} - - if eventType == "user.authentication.sso" or - eventType == "application.user_membership.add" or - eventType == "application.lifecycle.update" then - local target = log["target"] - if target and type(target) == "table" then - for _, t in ipairs(target) do - if type(t) == "table" then - if t["type"] == "AppInstance" then - log["dst_endpoint_uid"] = t["type"] - log["dst_endpoint_svc_name"] = t["displayName"] - elseif t["type"] == "AppUser" then - -- Override specific fields with target values - log["actor_user_uid"] = t["type"] -- "AppUser" overrides actor.id - log["actor_email_addr"] = t["alternateId"] -- "unknown" for actor.email_addr - log["actor_user_name"] = t["displayName"] -- "John Cena" (same as original) - - -- Override actor.id for actor.user.uid mapping - if log["actor"] then - log["actor"]["id"] = t["type"] -- "AppUser" for actor.user.uid - -- Keep actor.alternateId as original for actor.user.email_addr mapping - -- Don't override actor.alternateId - let it stay as "john.cena@wwe.com" - end - end - end - end - end - elseif eventType == "user.lifecycle.activate" or - eventType == "user.lifecycle.create" or - eventType == "user.lifecycle.deactivate" then - local target = log["target"] - if target and type(target) == "table" and #target > 0 then - local t = target[1] - if type(t) == "table" then - targetFields["user_id"] = t["id"] - targetFields["user_email_addr"] = t["alternateId"] - targetFields["user_name"] = t["displayName"] - end - end - elseif eventType == "policy.evaluate_sign_on" then - local target = log["target"] - if target and type(target) == "table" then - for _, t in ipairs(target) do - if type(t) == "table" then - if t["type"] == "PolicyEntity" then - targetFields["actor"] = { - authorization = { - policy = { - name = t["displayName"], - uid = t["id"] - } - } - } - elseif t["type"] == "PolicyRule" then - targetFields["actor"] = { - authorization = { - policy = { - rule = { - uid = t["id"], - name = t["displayName"] - } - } - } - } - end - end - end - end - end - - return targetFields -end - --- Generate severity mapping -function generateSeverityMapping(availableSeverityList) - local defaultSeverityMapping = { - ["DEBUG"] = 0, ["INFO"] = 1, ["WARN"] = 3, ["ERROR"] = 5, ["OTHER"] = 99 - } - local defaultSeverityMappingKeys = {"DEBUG", "INFO", "WARN", "ERROR", "OTHER"} - local severityIDMapping = {} - - for severityTypeIndex = 1, #availableSeverityList do - if availableSeverityList[severityTypeIndex] then - local key = defaultSeverityMappingKeys[severityTypeIndex] - severityIDMapping[key] = defaultSeverityMapping[key] - end - end - return severityIDMapping -end - --- Get severity ID -function getSeverityID(eventType, logSeverity) - local severityMapping = { - ["user.session.start"] = generateSeverityMapping({true, true, true, true, true}), - ["user.authentication.sso"] = generateSeverityMapping({true, true, true, false, false}), - ["user.lifecycle.activate"] = generateSeverityMapping({true, true, true, false, false}), - ["user.lifecycle.create"] = generateSeverityMapping({false, true, false, false, false}), - ["user.lifecycle.deactivate"] = generateSeverityMapping({false, true, false, false, false}), - ["policy.evaluate_sign_on"] = generateSeverityMapping({false, true, false, false, false}), - ["system.org.rate_limit.warning"] = generateSeverityMapping({true, true, true, false, false}), - ["application.user_membership.add"] = generateSeverityMapping({true, true, true, false, false}), - ["application.lifecycle.update"] = generateSeverityMapping({false, true, false, false, false}) - } - local eventSeverityMapping = severityMapping[eventType] or {} - return eventSeverityMapping[logSeverity] or 99 -end - --- Generate synthetic fields -function generateSyntheticFields(log, eventType, originalLog) - -- Set nested metadata fields - if not log["metadata"] then log["metadata"] = {} end - if not log["metadata"]["product"] then log["metadata"]["product"] = {} end - log["metadata"]["product"]["vendor_name"] = "Okta" - log["metadata"]["product"]["name"] = "Okta" - log["metadata"]["version"] = "1.0.0" - - local publishedTime = safelyAccessNestedDictKeys({"published"}, originalLog) - if publishedTime then - log["time"] = convertToMilliseconds(publishedTime) - end - - -- Use the event type passed as parameter instead of finding it again - log["category_name"] = getCategoryMapper(eventType) - log["category_uid"] = getCategoryUID(eventType) - - local className, classUid = getClassMapping(eventType) - log["class_name"] = className - log["class_uid"] = classUid - - -- Dynamic severity calculation - log["severity_id"] = getSeverityID(eventType, originalLog["severity"]) - - local activityName, activityId = getActivityName(eventType, originalLog["displayMessage"] or "Other") - log["activity_name"] = activityName - log["activity_id"] = activityId - - log["type_uid"] = (classUid * 100) + activityId - log["type_name"] = getTypeName(eventType) - - local outcomeResult = safelyAccessNestedDictKeys({"outcome", "result"}, originalLog) - log["status_id"] = getStatusDefaultOCSFMapping(outcomeResult) - - local actorType = safelyAccessNestedDictKeys({"actor", "type"}, originalLog) - - if not log["actor"] then log["actor"] = {} end - if not log["actor"]["user"] then log["actor"]["user"] = {} end - log["actor"]["user"]["type_id"] = getUserType(actorType) - log["actor"]["user"]["type"] = actorType or "User" - -- Actor email_addr is now set by target field mappings, don't override - - -- Handle postal code conversion to string - local postalCode = safelyAccessNestedDictKeys({"client", "geographicalContext", "postalCode"}, log) - if postalCode then - if not log["client"] then log["client"] = {} end - if not log["client"]["geographicalContext"] then log["client"]["geographicalContext"] = {} end - log["client"]["geographicalContext"]["postalCode"] = tostring(postalCode) - end - - -- Target fields are now processed earlier in oktaLogsMapping - - - log["event.type"] = log["activity_name"] or eventType - log["dataSource"] = {name = "Okta", category = "security", vendor = "Okta"} - - -- Add session field (skip root session for lifecycle events except deactivate) - local sessionId = safelyAccessNestedDictKeys({"authenticationContext", "externalSessionId"}, originalLog) - if not (eventType == "user.lifecycle.activate" or eventType == "user.lifecycle.create") then - if eventType == "user.lifecycle.deactivate" then - -- For deactivate events, add session to existing actor object if it exists - if log["actor"] then - log["actor"]["session"] = {uid = sessionId or "unknown"} - else - log["session"] = {uid = sessionId or "unknown"} - end - end - end - - -- Add user field from actor only for non-lifecycle events - --local userName = safelyAccessNestedDictKeys({"actor", "displayName"}, originalLog) - --if userName and not (eventType == "user.lifecycle.activate" or eventType == "user.lifecycle.create" or eventType == "user.lifecycle.deactivate" or eventType == "user.session.start") then - -- log["user"] = {name = userName} - --end - - return log -end - - --- Helper function to check if a field should be ignored -function shouldIgnoreField(fieldName) - - local ignoreFields = { - "_okta_event_type", "_ob", "ts", "timestamp" - } - - for _, field in ipairs(ignoreFields) do - if fieldName == field or string.find(fieldName, "^" .. field .. "%.") then - return true - end - end - return false -end - --- Helper function to check if a value is empty (empty object, array, or null) -function isEmptyValue(value) - if value == nil or value == "NULL_PLACEHOLDER" then - return true - end - if type(value) == "string" and (value == "" or value == "null") then - return true - end - if type(value) == "table" then - -- Check if it's an empty table - if next(value) == nil then - return true - end - -- For unmapped fields, recursively check nested values to filter out empty nested objects - local hasNonEmptyValues = false - for k, v in pairs(value) do - if not isEmptyValue(v) then - hasNonEmptyValues = true - break - end - end - return not hasNonEmptyValues - end - return false -end - --- Helper function to add unmapped fields as a truly nested object (no dotted keys) -function addUnmappedFields(sourceObj, targetObj, mappedFields, prefixParts) - prefixParts = prefixParts or {} - for key, value in pairs(sourceObj) do - -- Skip ignored fields - if shouldIgnoreField(key) then - goto continue - end - - -- Build current dotted path for mapped check - local currentPathParts = {} - for i = 1, #prefixParts do currentPathParts[i] = prefixParts[i] end - table.insert(currentPathParts, key) - local currentPath = table.concat(currentPathParts, ".") - - -- Check if this exact path has been mapped - local isMapped = mappedFields[currentPath] or false - - if not isMapped and value ~= nil then - if type(value) == "table" then - local nestedObj = {} - addUnmappedFields(value, nestedObj, mappedFields, currentPathParts) - - -- Check if ALL direct children were mapped - local allChildrenMapped = true - for childKey, childValue in pairs(value) do - local childPath = currentPath .. "." .. childKey - if not mappedFields[childPath] then - allChildrenMapped = false - break - end - end - - -- Only add parent if it has unmapped children OR no children were mapped - if next(nestedObj) and not allChildrenMapped then - targetObj[key] = nestedObj - end - -- Don't add empty objects to unmapped - else - -- Only add non-empty values to unmapped - if not isEmptyValue(value) then - targetObj[key] = value - end - end - end - ::continue:: - end -end - --- Helper function to build nested object structure from flat dotted keys -function buildNestedStructure(flatObj) - local nested = {} - for key, value in pairs(flatObj) do - local keys = split(key, ".") - local current = nested - for i = 1, #keys - 1 do - local k = keys[i] - if not current[k] then - current[k] = {} - elseif type(current[k]) ~= "table" then - -- If the existing value is not a table, we can't create nested structure - -- Skip this key to avoid conflicts - goto continue - end - current = current[k] - end - - -- Special handling for raw_data field - encode as JSON string - if keys[#keys] == "raw_data" then - -- For raw_data, we want the value as a JSON string, not double-encoded - if type(value) == "table" then - current[keys[#keys]] = encodeJson(value, "raw_data") - else - -- If it's already a string, use it as-is - current[keys[#keys]] = tostring(value) - end - else - current[keys[#keys]] = value - end - ::continue:: - end - return nested -end - --- Helper function to set nested value (marks field as processed) -function setNestedValue(obj, keys, value) - local current = obj - for i = 1, #keys - 1 do - if not current[keys[i]] then - current[keys[i]] = {} - end - current = current[keys[i]] - end - current[keys[#keys]] = value -end - - --- Helper function to filter out ignored fields using shouldIgnoreField -function filterIgnoredFields(log) - local function filterObject(obj, prefix) - prefix = prefix or "" - local filteredObj = {} - - for key, value in pairs(obj) do - local fullKey = prefix == "" and key or prefix .. "." .. key - - if not shouldIgnoreField(fullKey) then - if type(value) == "table" then - local filteredValue = filterObject(value, fullKey) - if next(filteredValue) then -- Only add non-empty tables - filteredObj[key] = filteredValue - end - else - filteredObj[key] = value - end - end - end - return filteredObj - end - - return filterObject(log) -end - - --- Helper function to create ordered JSON message using FIELD_ORDER approach -function createOrderedMessage(log, eventType) - -- Filter out null values and empty strings from the log before creating message - local filteredLog = {} - for k, v in pairs(log) do - if v ~= nil and v ~= "" and v ~= "null" then - if type(v) == "table" then - local filteredValue = filterNullValues(v) - if next(filteredValue) then -- Only add non-empty tables - filteredLog[k] = filteredValue - end - else - filteredLog[k] = v - end - end - end - - -- Apply domain filtering to the message for lifecycle events only - if eventType == "user.lifecycle.create" or eventType == "user.lifecycle.deactivate" or eventType == "user.session.start" then - filteredLog = applyDomainFiltering(filteredLog) - end - - -- Use FIELD_ORDERS.root to create ordered JSON string - local orderedJson = encodeWithFieldOrder(filteredLog, FIELD_ORDERS.root) - return orderedJson or "{}" -end - --- Helper function to recursively filter out null values from nested tables -function filterNullValues(obj) - local filtered = {} - for k, v in pairs(obj) do - if v ~= nil and v ~= "" and v ~= "null" then - if type(v) == "table" then - local filteredValue = filterNullValues(v) - if next(filteredValue) then -- Only add non-empty tables - filtered[k] = filteredValue - end - else - filtered[k] = v - end - end - end - return filtered -end - --- Helper function to apply domain filtering recursively -function applyDomainFiltering(obj) - if type(obj) ~= "table" then - return obj - end - - local filtered = {} - for k, v in pairs(obj) do - if type(v) == "table" then - local filteredValue = applyDomainFiltering(v) - if next(filteredValue) then -- Only add non-empty tables - filtered[k] = filteredValue - end - else - -- Skip domain fields with value "." and isp fields that duplicate asOrg - if k == "domain" and v == "." then - -- Skip this field - elseif k == "isp" and obj["asOrg"] and v == obj["asOrg"] then - -- Skip isp field when it's the same as asOrg - else - filtered[k] = v - end - end - end - return filtered -end - --- Simplified Okta logs mapping function -function oktaLogsMapping(log) - -- STEP 1: First filter out ignored fields using shouldIgnoreField - log = filterIgnoredFields(log) - - -- STEP 2: Find event type first - local eventType = findEventType(log) - - -- STEP 3: Create ordered JSON message from original log BEFORE any modifications - local originalLog = {} - for k, v in pairs(log) do - originalLog[k] = v - end - log.message = createOrderedMessage(originalLog, eventType) - - -- STEP 4: Process target fields to create flat keys for mapping - local targetFields = processTargetFields(log, eventType) - - -- Apply target fields to log for specific events so mappings can consume them - if eventType == "policy.evaluate_sign_on" then - for key, value in pairs(targetFields) do - if key == "actor" then - if not log["actor"] then log["actor"] = {} end - for subKey, subValue in pairs(value) do - log["actor"][subKey] = subValue - end - else - log[key] = value - end - end - elseif eventType == "user.lifecycle.activate" or eventType == "user.lifecycle.create" or eventType == "user.lifecycle.deactivate" then - -- For lifecycle events, expose target-derived user fields on the log - for key, value in pairs(targetFields) do - log[key] = value - end - end - - -- STEP 4: Get mapping for event type - local mappings = getDefaultMapping(eventType) - - -- Merge with common mapping - local commonMappings = getCommonMapping() - for _, mapping in ipairs(commonMappings) do - table.insert(mappings, mapping) - end - - -- STEP 4: Apply mappings and track processed fields - local parsedData = {} - local mappedFields = {} - for _, mapping in ipairs(mappings) do - local sourcePath = mapping.source - local targetPath = mapping.target - local keys = split(sourcePath, ".") - local value = safelyAccessNestedDictKeys(keys, log) - - if value ~= nil and value ~= "" and value ~= "null" then - if targetPath == "src_endpoint.location.domain" and value == "." then - -- Skip this field for any event when domain is a single dot - else - -- Update target in parsedData - parsedData[targetPath] = value - -- Track mapped field by source path - mappedFields[sourcePath] = true - end - end - end - - -- STEP 5: Add remaining non-null fields to unmapped - parsedData["unmapped"] = {} - local ignoreUnmappedFields = { - "actor_user_uid", - "actor_email_addr", - "actor_user_name", - "user_id", - "user_email_addr", - "user_name" - } - for _, field in ipairs(ignoreUnmappedFields) do - mappedFields[field] = true - end - - addUnmappedFields(log, parsedData["unmapped"], mappedFields, {}) - - -- Apply domain filtering to unmapped fields as well - parsedData["unmapped"] = applyDomainFiltering(parsedData["unmapped"]) - - -- STEP 6: Convert flat dotted keys to nested objects - parsedData = buildNestedStructure(parsedData) - - -- STEP 7: Apply post-processing - parsedData = parseSecurityDomain(parsedData) - parsedData = parseRiskLevel(parsedData) - - -- STEP 8: Generate synthetic fields - parsedData = generateSyntheticFields(parsedData, eventType, log) - - -- STEP 9: Generate observables for specific event types (after synthetic fields) - local observablesEventTypes = { - "user.authentication.sso", - "application.user_membership.add", - "user.lifecycle.activate", - "user.lifecycle.create", - "user.lifecycle.deactivate", - "policy.evaluate_sign_on", - "system.org.rate_limit.warning", - "user.session.start" - } - - for _, eventTypeName in ipairs(observablesEventTypes) do - if eventType == eventTypeName then - local observables = getObservables(log) - -- Convert observables to array format - local observablesArray = {} - for _, obs in ipairs(observables) do - table.insert(observablesArray, obs) - end - parsedData["observables"] = observablesArray - break - end - end - - parsedData = convertToNested(parsedData, eventType) - return parsedData -end - --- Convert flat parsed data to nested JSON structure with field ordering -function convertToNested(parsedData, eventType) - local nested = {} - - -- First, build the nested structure - for key, value in pairs(parsedData) do - -- Special case: keep event.type as flattened field (don't nest it) - if key == "event.type" then - nested["event.type"] = value -- Keep as flattened event.type - goto continue - end - - local keys = split(key, ".") - local current = nested - - -- Navigate to the parent object - for i = 1, #keys - 1 do - local k = keys[i] - if not current[k] then - current[k] = {} - elseif type(current[k]) ~= "table" then - -- If the existing value is not a table, we can't create nested structure - -- Skip this key to avoid conflicts - goto continue - end - current = current[k] - end - - -- Set the final value - special handling for raw_data field - if keys[#keys] == "raw_data" then - -- For raw_data, we want the value as a JSON string, not double-encoded - if type(value) == "table" then - current[keys[#keys]] = encodeJson(value, "raw_data") - else - -- If it's already a string, use it as-is - current[keys[#keys]] = tostring(value) - end - else - current[keys[#keys]] = value - end - ::continue:: - end - - -- Then, apply field ordering to the nested structure - return applyFieldOrdering(nested, nil, eventType) -end - --- Apply field ordering to nested structure -function applyFieldOrdering(obj, fieldOrder, eventType) - fieldOrder = fieldOrder or FIELD_ORDERS.root - local ordered = {} - - -- Phase 1: Add fields in predefined order - for _, fieldName in ipairs(fieldOrder) do - if obj[fieldName] ~= nil then - if type(obj[fieldName]) == "table" then - -- Recursively apply ordering to nested objects - local nestedFieldOrder = FIELD_ORDERS[fieldName] - ordered[fieldName] = applyFieldOrdering(obj[fieldName], nestedFieldOrder, eventType) - else - ordered[fieldName] = obj[fieldName] - end - end - end - - -- Phase 2: Add remaining fields not in the predefined order - for key, value in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if key == fieldName then - found = true - break - end - end - - if not found then - if type(value) == "table" then - -- Recursively apply ordering to nested objects - local nestedFieldOrder = FIELD_ORDERS[key] - ordered[key] = applyFieldOrdering(value, nestedFieldOrder, eventType) - else - ordered[key] = value - end - end - end - - return ordered -end - --- Convert ISO 8601 timestamp to Unix epoch milliseconds -function convertToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - - -- Parse ISO 8601 format: "2025-09-29T09:15:40Z" or "2025-09-29T09:15:40.123Z" - local year, month, day, hour, min, sec, ms = string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z") - - if year and month and day and hour and min and sec then - local t = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - - -- Get local time interpretation - local local_seconds = os.time(t) - - -- Get what this represents in UTC - local utc_date = os.date("!*t", local_seconds) - - local unix_seconds - -- Check if the UTC interpretation matches our input - if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and - utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and - utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then - -- We are already in UTC, use as-is - unix_seconds = local_seconds - else - -- Calculate the correct UTC timestamp - local utc_seconds = os.time(utc_date) - local offset = local_seconds - utc_seconds - unix_seconds = local_seconds + offset -- Add offset to get UTC - end - - -- Add milliseconds if present - local milli = 0 - if ms and ms ~= "" then - milli = tonumber((ms .. "000"):sub(1, 3)) -- pad/truncate to 3 digits - end - - return unix_seconds * 1000 + milli - end - - return nil -end - --- Main event processing function -function processEvent(event) - return oktaLogsMapping(event) -end \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/wiz_issue/metadata.yaml b/pipelines/community/transform_ocsf/wiz_issue/metadata.yaml deleted file mode 100644 index 9699c5a..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/metadata.yaml +++ /dev/null @@ -1,52 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Wiz Issue. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: wiz - dataSource: Wiz Issue - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "API Key & Secret" - sample_record: "{\n \"id\": \"wiz-issue-92324e96-d8d1-4e9a-bcb8-14eed356e1a5\",\n \"targetExternalId\"\ - : \"arn:aws:s3:::company-public-assets\",\n \"deleted\": false,\n \"targetObjectProviderUniqueId\"\ - : \"arn:aws:s3:::company-public-assets\",\n \"firstSeenAt\": \"2026-04-19T14:51:33.000Z\",\n \"\ - result\": \"FAIL\",\n \"status\": \"REJECTED\",\n \"severity\": \"MEDIUM\",\n \"remediation\":\ - \ \"Update bucket ACL to remove public access\",\n \"resource\": {\n \"id\": \"resource-8b1a9118-bd96-48fa-9661-c8fef99bc28c\"\ - ,\n \"providerId\": \"aws-account-123456789012\",\n \"name\": \"company-public-assets\",\n \ - \ \"nativeType\": \"S3 Bucket\",\n \"type\": \"BUCKET\",\n \"region\": \"ap-northeast-1\"\ - ,\n \"subscription\": {\n \"id\": \"sub-aws-prod-001\",\n \"name\": \"AWS Production\ - \ Account\",\n \"externalId\": \"123456789012\",\n \"cloudProvider\": \"AWS\"\n },\n\ - \ \"projects\": [\n {\n \"id\": \"project-marketing-001\",\n \"name\": \"Marketing\ - \ Assets\",\n \"riskProfile\": {\n \"businessImpact\": \"MBI\"\n }\n }\n\ - \ ],\n \"tags\": [\n {\n \"key\": \"Environment\",\n \"value\": \"Production\"\ - \n },\n {\n \"key\": \"Department\",\n \"value\": \"Marketing\"\n }\n\ - \ ]\n },\n \"rule\": {\n \"id\": \"rule-s3-public-001\",\n \"graphId\": \"graph-rule-001\"\ - ,\n \"name\": \"S3 bucket is publicly accessible\",\n \"description\": \"S3 bucket allows public\ - \ read access\",\n \"remediationInstructions\": \"Remove public ACL grants from bucket policy\"\ - ,\n \"functionAsControl\": false\n },\n \"securitySubCategories\": [\n {\n \"id\":" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: wiz, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/wiz_issue/sample.json b/pipelines/community/transform_ocsf/wiz_issue/sample.json deleted file mode 100644 index 5666680..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/sample.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "id": "wiz-issue-92324e96-d8d1-4e9a-bcb8-14eed356e1a5", - "targetExternalId": "arn:aws:s3:::company-public-assets", - "deleted": false, - "targetObjectProviderUniqueId": "arn:aws:s3:::company-public-assets", - "firstSeenAt": "2026-04-19T14:51:33.000Z", - "result": "FAIL", - "status": "REJECTED", - "severity": "MEDIUM", - "remediation": "Update bucket ACL to remove public access", - "resource": { - "id": "resource-8b1a9118-bd96-48fa-9661-c8fef99bc28c", - "providerId": "aws-account-123456789012", - "name": "company-public-assets", - "nativeType": "S3 Bucket", - "type": "BUCKET", - "region": "ap-northeast-1", - "subscription": { - "id": "sub-aws-prod-001", - "name": "AWS Production Account", - "externalId": "123456789012", - "cloudProvider": "AWS" - }, - "projects": [ - { - "id": "project-marketing-001", - "name": "Marketing Assets", - "riskProfile": { - "businessImpact": "MBI" - } - } - ], - "tags": [ - { - "key": "Environment", - "value": "Production" - }, - { - "key": "Department", - "value": "Marketing" - } - ] - }, - "rule": { - "id": "rule-s3-public-001", - "graphId": "graph-rule-001", - "name": "S3 bucket is publicly accessible", - "description": "S3 bucket allows public read access", - "remediationInstructions": "Remove public ACL grants from bucket policy", - "functionAsControl": false - }, - "securitySubCategories": [ - { - "id": "cat-data-exposure-001", - "title": "Data Exposure", - "category": { - "id": "cat-parent-001", - "name": "Data Protection", - "framework": { - "id": "framework-cis-001", - "name": "CIS AWS Foundations Benchmark" - } - } - } - ], - "ignoreRules": null -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/wiz_issue/serializer.lua b/pipelines/community/transform_ocsf/wiz_issue/serializer.lua deleted file mode 100644 index 9a94bba..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/serializer.lua +++ /dev/null @@ -1,632 +0,0 @@ - -local FEATURES = { - FLATTEN_EVENT_TYPE = true, -} - --- Wiz to OCSF Mapping Script -local OCSF_VERSION = "v1.0.0-rc.3" - -local WIZ_SEVERITY_MAP = {["INFORMATIONAL"]=1, ["LOW"]=2, ["MEDIUM"]=3, ["HIGH"]=4, ["CRITICAL"]=5, ["OTHER"]=99} -local WIZ_STATE_MAP = {["OPEN"]=1, ["IN_PROGRESS"]=2, ["REJECTED"]=3, ["RESOLVED"]=4, ["OTHER"]=99} -local OCSF_ACCOUNT_TYPE_ID = {["AWS"]=10, ["GCP"]=5, ["EKS"]=10, ["GKE"]=5} -local OCSF_ACCOUNT_TYPE_NAME = {[10]="AWS Account", [5]="GCP Account", [3]="AWS IAM User", [4]="AWS IAM Role"} - --- Field ordering templates for consistent JSON serialization -local FIELD_ORDERS = { - root = {"activity_id", "analytic", "category_name", "category_uid", "class_name", "class_uid", - "cloud", "dataSource", "event", "finding", "index", "metadata", "message", "resource", - "severity", "severity_id", "state", "state_id", - "status", "type_uid", "unmapped" - }, - message = { - "activity_id", "analytic_type", "analytic_type_id", "category_name", "category_uid", - "class_name", "class_uid", "cloud_account_type", "cloud_account_type_id", "datasource", - "entitySnapshot", "event", "id", "ocsf_version", "severity", "severity_id", - "sourceRule", "state_id", "status", "type_uid", "updatedAt", "eventType", "readOnly", "createdAt" - }, - entitySnapshot = { - "cloudPlatform", "cloudProviderURL", "externalId", "id", "name", "nativeType", - "projects", "providerId", "region", "resourceGroupExternalId", "status", - "subscription", "subscriptionExternalId", "type", "wizResourceID" - }, - sourceRule = { - "description", "id", "name", "remediationInstructions", "resolutionRecommendation" - }, - datasource = { - "category", "name", "vendor" - }, - subscription = { - "externalId", "name", "id", "cloudProvider" - }, - projects = { - "name", "riskProfile", "id" - }, - riskProfile = { - "businessImpact" - }, - unmapped = { - "severity" - } -} - --- JSON encoding function (from AWS CloudTrail) -function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray and maxIndex > 0 then - local items = {} - for i = 1, maxIndex do - local elementKey = key or tostring(i) - table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or "null") - end - return "[" .. table.concat(items, ", ") .. "]" - else - local items = {} - local fieldOrder = FIELD_ORDERS[key] or {} - - -- Phase 1: Process fields in predefined order - for _, fieldName in ipairs(fieldOrder) do - local v = obj[fieldName] - if v ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '": ' .. encodeJson(v, fieldName)) - end - end - - -- Phase 2: Process remaining fields - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. encodeJson(v, keyStr)) - end - end - - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - --- Helper function to set nested fields (from AWS CloudTrail) -function setNestedField(obj, path, value) - if value == nil or path == nil or path == '' then return end - local keys = {} - for key in string.gmatch(path, '[^.]+') do - if key and key ~= '' then - table.insert(keys, key) - end - end - if #keys == 0 then return end - - local current = obj - for i = 1, #keys - 1 do - local key = keys[i] - if current[key] == nil then - current[key] = {} - end - current = current[key] - end - current[keys[#keys]] = value -end - -local IGNORE_FIELDS = { - _wiz_event_type = true, - _wiz_query_time = true, - _ts = true, - _ob = true, - host = true, - source_type = true, - timestamp = true, - type = true -} - -local fieldMappings = { - -- Finding and Analytic mappings - {type="multi", source="sourceRule.description", targets={"finding.desc", "analytic.desc"}}, - {type="multi", source="sourceRule.name", targets={"finding.title", "analytic.name"}}, - {type="direct", source="sourceRule.resolutionRecommendation", target="finding.remediation.desc"}, - {type="direct", source="sourceRule.id", target="analytic.uid"}, - {type="direct", source="analytic_type_id", target="analytic.type_id"}, - {type="direct", source="analytic_type", target="analytic.type"}, - {type="direct", source="createdAt", target="finding.created_time"}, - {type="direct", source="updatedAt", target="finding.modified_time"}, - - -- Cloud mappings - {type="direct", priority1="entitySnapshot.subscription.cloudProvider", priority2="entitySnapshot.cloudPlatform", target="cloud.provider"}, - {type="direct", source="cloud_account_type", target="cloud.account.type"}, - {type="direct", source="cloud_account_type_id", target="cloud.account.type_id"}, - {type="direct", source="entitySnapshot.region", target="cloud.region"}, - {type="multi", priority1="entitySnapshot.subscription.externalId", priority2="entitySnapshot.subscriptionExternalId", targets={"cloud.account.uid", "cloud.org.uid"}}, - - -- Resource mappings - {type="direct", source="entitySnapshot.cloudProviderURL", target="resource.url"}, - {type="direct", source="entitySnapshot.externalId", target="resource.data.external_id"}, - {type="direct", source="entitySnapshot.wizResourceID.id", target="resource.data.wiz_resource_id"}, - {type="direct", source="entitySnapshot.name", target="resource.name"}, - {type="direct", source="entitySnapshot.nativeType", target="resource.type"}, - {type="direct", source="entitySnapshot.providerId", target="resource.uid"}, - {type="direct", source="entitySnapshot.resourceGroupExternalId", target="resource.group.uid"}, - {type="direct", source="entitySnapshot.status", target="resource.status"}, - {type="direct", source="entitySnapshot.type", target="resource.data.generic_type"}, - - -- Metadata and event mappings - {type="multi", source="id", targets={"metadata.uid", "finding.uid"}}, - {type="direct", source="event", target="event.type"}, - {type="direct", source="severity", target="unmapped.severity"}, - {type="multi", source="status", targets={"state", "status"}}, - {type="multi", source="datasource.vendor", targets={"dataSource.vendor", "metadata.product.vendor_name"}}, - {type="multi", source="datasource.name", targets={"dataSource.name", "metadata.product.name"}}, - {type="direct", source="datasource.category", target="dataSource.category"}, - {type="direct", source="ocsf_version", target="metadata.version"}, - - -- OCSF standard fields (computed values) - {type="computed", value="2001", target="class_uid"}, - {type="computed", value="99", target="activity_id"}, - {type="computed", value="200199", target="type_uid"}, - {type="computed", value="2", target="category_uid"}, - {type="computed", value="Security Finding", target="class_name"}, - {type="computed", value="Findings", target="category_name"}, - - -- Additional required fields - - -- Computed mappings using maps - {type="computed_map", source="severity", map="WIZ_SEVERITY_MAP", default=99, target="severity_id"}, - {type="computed_map", source="status", map="WIZ_STATE_MAP", default=99, target="state_id"}, - - -- Cloud account type mappings - {type="computed_cloud_type", source="entitySnapshot.cloudPlatform", target="cloud.account.type_id"}, - {type="computed_cloud_type", source="entitySnapshot.cloudPlatform", target="cloud.account.type"}, - -} - - --- Field ordering is now defined by the mapping order in fieldMappings table - -function getNestedField(obj, path) - if not obj or not path or path == '' then return nil end - local current = obj - for key in string.gmatch(path, '[^.]+') do - if not current or not key then return nil end - current = current[key] - end - return current -end - -function processEvent(event) - if not event then return event end - - -- Populate event.event from _wiz_event_type if it's an issue event - if event._wiz_event_type and string.lower(event._wiz_event_type) == "issue" then - event.event = "Issues" - end - - -- Check if this is an issue event (case-insensitive) - local isIssueEvent = false - if event.event and string.lower(event.event) == "issues" then - isIssueEvent = true - end - - if not isIssueEvent then - -- Return as-is for non-issue events - return event - end - - -- Add datasource field to event - if not event.datasource then - event.datasource = { - category = "security", - name = "Wiz", - vendor = "Wiz" - } - end - - -- Set event field - event.event = "Issues" - - -- Add OCSF fields - event.class_uid = "2001" - event.activity_id = "99" - event.type_uid = "200199" - event.category_uid = "2" - event.class_name = "Security Finding" - event.category_name = "Findings" - event.analytic_type = "Rule" - event.analytic_type_id = 1 - - -- Add computed fields - event.severity_id = WIZ_SEVERITY_MAP[event.severity] or 99 - event.state_id = WIZ_STATE_MAP[event.status] or 99 - - - -- Add cloud account type_id (always show) and type (only if cloudPlatform exists) - local cloudPlatform = nil - if event.entitySnapshot and event.entitySnapshot.cloudPlatform then - cloudPlatform = event.entitySnapshot.cloudPlatform - end - - event.cloud_account_type_id = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99 - - -- Check for AWS IAM users/roles - if event.cloud_account_type_id == 10 and event.entitySnapshot and event.entitySnapshot.nativeType then - local nativeType = event.entitySnapshot.nativeType - if nativeType == "user" then - event.cloud_account_type_id = 3 -- AWS IAM User - elseif nativeType == "role" then - event.cloud_account_type_id = 4 -- AWS IAM Role - end - end - - -- Only set cloud_account_type if cloudPlatform exists - if cloudPlatform then - event.cloud_account_type = OCSF_ACCOUNT_TYPE_NAME[event.cloud_account_type_id] or cloudPlatform or "Unknown Account Type" - end - -- If cloudPlatform is nil, don't set cloud_account_type but type_id still shows - - -- Add wizResourceID to entitySnapshot - if event.entitySnapshot and event.entitySnapshot.id then - event.entitySnapshot.wizResourceID = { - id = event.entitySnapshot.id - } - end - - -- Convert timestamps to Unix milliseconds - if event.createdAt then - -- Convert createdAt to Unix milliseconds - local createdAtStr = tostring(event.createdAt) - -- Try to parse as ISO timestamp and convert to milliseconds - local success, result = pcall(function() - -- Simple conversion for ISO timestamps like "2025-10-08T22:12:42.66996Z" - if createdAtStr:match("^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d") then - -- Extract the timestamp part and convert - local year, month, day, hour, min, sec = createdAtStr:match("^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)") - if year and month and day and hour and min and sec then - -- Create a simple timestamp (this is a basic implementation) - -- In a real implementation, you'd use proper date parsing - local timestamp = os.time({ - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec) - }) - return timestamp * 1000 -- Convert to milliseconds - end - end - -- Fallback: try to convert as number - local num = tonumber(createdAtStr) - if num then - -- If it's already a timestamp, ensure it's in milliseconds - if num > 1000000000000 then -- Already in milliseconds - return num - elseif num > 1000000000 then -- In seconds, convert to milliseconds - return num * 1000 - else -- Assume it's already in the right format - return num - end - end - return nil - end) - - if success and result then - event.createdAt = result - end - end - - if event.updatedAt then - -- Convert updatedAt to Unix milliseconds - local updatedAtStr = tostring(event.updatedAt) - local success, result = pcall(function() - -- Simple conversion for ISO timestamps - if updatedAtStr:match("^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d") then - local year, month, day, hour, min, sec = updatedAtStr:match("^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)") - if year and month and day and hour and min and sec then - local timestamp = os.time({ - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec) - }) - return timestamp * 1000 - end - end - local num = tonumber(updatedAtStr) - if num then - if num > 1000000000000 then - return num - elseif num > 1000000000 then - return num * 1000 - else - return num - end - end - return nil - end) - - if success and result then - event.updatedAt = result - end - end - - -- Set OCSF version - event.ocsf_version = OCSF_VERSION - - -- Remove tags field from entitySnapshot - if event.entitySnapshot then - event.entitySnapshot.tags = nil - end - - -- Track all fields that were added to the event as mapped - -- so they don't appear in unmapped - local mappedFields = {} - mappedFields["class_uid"] = true - mappedFields["activity_id"] = true - mappedFields["type_uid"] = true - mappedFields["category_uid"] = true - mappedFields["class_name"] = true - mappedFields["category_name"] = true - mappedFields["analytic_type"] = true - mappedFields["analytic_type_id"] = true - mappedFields["severity_id"] = true - mappedFields["state_id"] = true - mappedFields["cloud_account_type_id"] = true - mappedFields["cloud_account_type"] = true - mappedFields["ocsf_version"] = true - mappedFields["event"] = true - mappedFields["entitySnapshot"] = true - - local result = {} - - -- Track field order as they're processed - local fieldOrder = {} - local processedFields = {} - - -- Helper functions - local function getValue(priority1, priority2, priority3) - local value = getNestedField(event, priority1) - if value then return value end - if priority2 then - value = getNestedField(event, priority2) - if value then return value end - end - if priority3 then - return getNestedField(event, priority3) - end - return nil - end - - local function setValue(targetPath, value) - if value == "NULL_PLACEHOLDER" or value == "-" or value == "" then - setNestedField(result, targetPath, nil) - elseif value ~= nil then - setNestedField(result, targetPath, value) - end - end - - local function setMultiValue(targetPaths, value) - if value == "NULL_PLACEHOLDER" or value == "-" or value == "" then - for _, targetPath in ipairs(targetPaths) do - setNestedField(result, targetPath, nil) - end - elseif value then - for _, targetPath in ipairs(targetPaths) do - setNestedField(result, targetPath, value) - end - end - end - - -- Track field order as they're processed - local fieldOrder = {} - local processedFields = {} - - -- Apply mappings - for _, mapping in ipairs(fieldMappings) do - local value = nil - - if mapping.type == "computed" then - value = mapping.value - elseif mapping.type == "computed_map" then - -- Computed value from source using map - local sourceValue = getNestedField(event, mapping.source) - local mapTable = nil - if mapping.map == "WIZ_SEVERITY_MAP" then - mapTable = WIZ_SEVERITY_MAP - elseif mapping.map == "WIZ_STATE_MAP" then - mapTable = WIZ_STATE_MAP - end - value = (mapTable and mapTable[sourceValue]) or mapping.default - elseif mapping.type == "computed_cloud_type" then - -- Handle cloud account type logic with AWS IAM support - local cloudPlatform = getNestedField(event, mapping.source) - local typeId = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99 - - -- Check for AWS IAM users/roles - if typeId == 10 and getNestedField(event, "entitySnapshot.nativeType") then - local nativeType = getNestedField(event, "entitySnapshot.nativeType") - if nativeType == "user" then - typeId = 3 -- AWS IAM User - elseif nativeType == "role" then - typeId = 4 -- AWS IAM Role - end - end - - if mapping.target == "cloud.account.type_id" or mapping.target == "message.cloud_account_type_id" then - value = typeId -- Always show type_id - elseif mapping.target == "cloud.account.type" or mapping.target == "message.cloud_account_type" then - -- Only show type if cloudPlatform exists - if cloudPlatform then - value = OCSF_ACCOUNT_TYPE_NAME[typeId] or cloudPlatform or "Unknown Account Type" - end - -- If cloudPlatform is nil, value remains nil (field will be ignored) - end - elseif mapping.type == "message_field" then - value = getNestedField(event, mapping.source) - elseif mapping.priority1 and mapping.priority2 then - value = getValue(mapping.priority1, mapping.priority2, mapping.priority3) - else - value = getNestedField(event, mapping.source) - end - - -- Set the value and track order - if mapping.type == "direct" then - setValue(mapping.target, value) - if not processedFields[mapping.target] then - table.insert(fieldOrder, mapping.target) - processedFields[mapping.target] = true - end - elseif mapping.type == "multi" then - setMultiValue(mapping.targets, value) - for _, target in ipairs(mapping.targets) do - if not processedFields[target] then - table.insert(fieldOrder, target) - processedFields[target] = true - end - end - elseif mapping.type == "computed" or mapping.type == "computed_map" then - setValue(mapping.target, value) - if not processedFields[mapping.target] then - table.insert(fieldOrder, mapping.target) - processedFields[mapping.target] = true - end - elseif mapping.type == "computed_cloud_type" then - if mapping.target:match("^message%.") then - -- Handle message fields separately - if not result["message"] then - result["message"] = {} - end - setNestedField(result["message"], mapping.target:gsub("message%.", ""), value) - else - setValue(mapping.target, value) - if not processedFields[mapping.target] then - table.insert(fieldOrder, mapping.target) - processedFields[mapping.target] = true - end - end - end - - -- Track mapped fields for unmapped processing - if mapping.source then - mappedFields[mapping.source] = true - end - if mapping.priority1 then - mappedFields[mapping.priority1] = true - end - if mapping.priority2 then - mappedFields[mapping.priority2] = true - end - if mapping.priority3 then - mappedFields[mapping.priority3] = true - end - - -- For computed fields, we need to track the target fields as mapped - -- since they will be added to the event and shouldn't appear in unmapped - if mapping.type == "computed" or mapping.type == "computed_map" or mapping.type == "computed_cloud_type" then - if mapping.target then - -- Extract the root field name from the target path - local rootField = mapping.target:match("^([^%.]+)") - if rootField then - mappedFields[rootField] = true - end - end - end - end - - -- Only include fields that are NOT mapped and NOT ignored - for key, value in pairs(event) do - if not IGNORE_FIELDS[key] and not mappedFields[key] then - if type(value) == 'table' then - local nestedObj = {} - for nestedKey, nestedValue in pairs(value) do - if type(nestedValue) ~= 'function' and nestedValue ~= "-" and nestedValue ~= "" then - -- Check if this nested field was mapped - local nestedFieldKey = key .. "." .. nestedKey - if not mappedFields[nestedFieldKey] then - nestedObj[nestedKey] = nestedValue - end - end - end - if next(nestedObj) then - result["unmapped." .. key] = nestedObj - end - elseif value ~= "-" and value ~= "" then - result["unmapped." .. key] = value - end - end - end - - -- Build nested structure instead of flattening - local nested = {} - - -- Add mapped fields to nested structure - for _, fieldName in ipairs(fieldOrder) do - local value = getNestedField(result, fieldName) - if value ~= nil then - setNestedField(nested, fieldName, value) - end - end - - -- Add message field as JSON string (like AWS CloudTrail does) - local cleanEvent = {} - for key, value in pairs(event) do - if key ~= "_ob" and key ~= "timestamp" and key ~= "_ts" and key ~= "_wiz_event_type" and key ~= "_wiz_query_time" and value ~= "-" and value ~= "" then - cleanEvent[key] = value - end - end - - nested["message"] = encodeJson(cleanEvent, "message") - - -- Add missing fields that should be in the message - if cleanEvent.readOnly == nil then - cleanEvent.readOnly = false -- Default for Wiz events - end - if cleanEvent.eventType == nil then - cleanEvent.eventType = "Issues" -- Default for Wiz events - end - - -- Add unmapped fields as nested object (using setNestedField like AWS CloudTrail) - for key, value in pairs(result) do - if key:match("^unmapped%.") then - local fieldName = key:gsub("^unmapped%.", "") - setNestedField(nested, "unmapped." .. fieldName, value) - end - end - - -- Add time field - if nested.finding and nested.finding.modified_time then - nested["time"] = nested.finding.modified_time - end - - if FEATURES.FLATTEN_EVENT_TYPE then - if nested and nested.event then - nested['event.type'] = nested.event.type - end - end - return nested -end - diff --git a/pipelines/community/transform_ocsf/wiz_issue/wiz_issue.json b/pipelines/community/transform_ocsf/wiz_issue/wiz_issue.json deleted file mode 100644 index 213162b..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/wiz_issue.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Wiz Issue", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Wiz Issue. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "wiz", - "source_name": "wiz_issue", - "version": "1.0.0-rc.3", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "wiz-issue-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "\nlocal FEATURES = {\n FLATTEN_EVENT_TYPE = true,\n}\n\n-- Wiz to OCSF Mapping Script\nlocal OCSF_VERSION = \"v1.0.0-rc.3\"\n\nlocal WIZ_SEVERITY_MAP = {[\"INFORMATIONAL\"]=1, [\"LOW\"]=2, [\"MEDIUM\"]=3, [\"HIGH\"]=4, [\"CRITICAL\"]=5, [\"OTHER\"]=99}\nlocal WIZ_STATE_MAP = {[\"OPEN\"]=1, [\"IN_PROGRESS\"]=2, [\"REJECTED\"]=3, [\"RESOLVED\"]=4, [\"OTHER\"]=99}\nlocal OCSF_ACCOUNT_TYPE_ID = {[\"AWS\"]=10, [\"GCP\"]=5, [\"EKS\"]=10, [\"GKE\"]=5}\nlocal OCSF_ACCOUNT_TYPE_NAME = {[10]=\"AWS Account\", [5]=\"GCP Account\", [3]=\"AWS IAM User\", [4]=\"AWS IAM Role\"}\n\n-- Field ordering templates for consistent JSON serialization\nlocal FIELD_ORDERS = {\n root = {\"activity_id\", \"analytic\", \"category_name\", \"category_uid\", \"class_name\", \"class_uid\",\n \"cloud\", \"dataSource\", \"event\", \"finding\", \"index\", \"metadata\", \"message\", \"resource\",\n \"severity\", \"severity_id\", \"state\", \"state_id\",\n \"status\", \"type_uid\", \"unmapped\"\n },\n message = {\n \"activity_id\", \"analytic_type\", \"analytic_type_id\", \"category_name\", \"category_uid\",\n \"class_name\", \"class_uid\", \"cloud_account_type\", \"cloud_account_type_id\", \"datasource\",\n \"entitySnapshot\", \"event\", \"id\", \"ocsf_version\", \"severity\", \"severity_id\",\n \"sourceRule\", \"state_id\", \"status\", \"type_uid\", \"updatedAt\", \"eventType\", \"readOnly\", \"createdAt\"\n },\n entitySnapshot = {\n \"cloudPlatform\", \"cloudProviderURL\", \"externalId\", \"id\", \"name\", \"nativeType\",\n \"projects\", \"providerId\", \"region\", \"resourceGroupExternalId\", \"status\",\n \"subscription\", \"subscriptionExternalId\", \"type\", \"wizResourceID\"\n },\n sourceRule = {\n \"description\", \"id\", \"name\", \"remediationInstructions\", \"resolutionRecommendation\"\n },\n datasource = {\n \"category\", \"name\", \"vendor\"\n },\n subscription = {\n \"externalId\", \"name\", \"id\", \"cloudProvider\"\n },\n projects = {\n \"name\", \"riskProfile\", \"id\"\n },\n riskProfile = {\n \"businessImpact\"\n },\n unmapped = {\n \"severity\"\n }\n}\n\n-- JSON encoding function (from AWS CloudTrail)\nfunction encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" then\n return \"null\"\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n \n if isArray and maxIndex > 0 then\n local items = {}\n for i = 1, maxIndex do\n local elementKey = key or tostring(i)\n table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or \"null\")\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n local items = {}\n local fieldOrder = FIELD_ORDERS[key] or {}\n \n -- Phase 1: Process fields in predefined order\n for _, fieldName in ipairs(fieldOrder) do\n local v = obj[fieldName]\n if v ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, fieldName))\n end\n end\n \n -- Phase 2: Process remaining fields\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then \n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, keyStr))\n end\n end\n \n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\n-- Helper function to set nested fields (from AWS CloudTrail)\nfunction setNestedField(obj, path, value)\n if value == nil or path == nil or path == '' then return end\n local keys = {}\n for key in string.gmatch(path, '[^.]+') do\n if key and key ~= '' then\n table.insert(keys, key)\n end\n end\n if #keys == 0 then return end\n \n local current = obj\n for i = 1, #keys - 1 do\n local key = keys[i]\n if current[key] == nil then\n current[key] = {}\n end\n current = current[key]\n end\n current[keys[#keys]] = value\nend\n\nlocal IGNORE_FIELDS = { \n _wiz_event_type = true, \n _wiz_query_time = true, \n _ts = true, \n _ob = true,\n host = true,\n source_type = true,\n timestamp = true,\n type = true\n}\n\nlocal fieldMappings = {\n -- Finding and Analytic mappings\n {type=\"multi\", source=\"sourceRule.description\", targets={\"finding.desc\", \"analytic.desc\"}},\n {type=\"multi\", source=\"sourceRule.name\", targets={\"finding.title\", \"analytic.name\"}},\n {type=\"direct\", source=\"sourceRule.resolutionRecommendation\", target=\"finding.remediation.desc\"},\n {type=\"direct\", source=\"sourceRule.id\", target=\"analytic.uid\"},\n {type=\"direct\", source=\"analytic_type_id\", target=\"analytic.type_id\"},\n {type=\"direct\", source=\"analytic_type\", target=\"analytic.type\"},\n {type=\"direct\", source=\"createdAt\", target=\"finding.created_time\"},\n {type=\"direct\", source=\"updatedAt\", target=\"finding.modified_time\"},\n \n -- Cloud mappings\n {type=\"direct\", priority1=\"entitySnapshot.subscription.cloudProvider\", priority2=\"entitySnapshot.cloudPlatform\", target=\"cloud.provider\"},\n {type=\"direct\", source=\"cloud_account_type\", target=\"cloud.account.type\"},\n {type=\"direct\", source=\"cloud_account_type_id\", target=\"cloud.account.type_id\"},\n {type=\"direct\", source=\"entitySnapshot.region\", target=\"cloud.region\"},\n {type=\"multi\", priority1=\"entitySnapshot.subscription.externalId\", priority2=\"entitySnapshot.subscriptionExternalId\", targets={\"cloud.account.uid\", \"cloud.org.uid\"}},\n \n -- Resource mappings\n {type=\"direct\", source=\"entitySnapshot.cloudProviderURL\", target=\"resource.url\"},\n {type=\"direct\", source=\"entitySnapshot.externalId\", target=\"resource.data.external_id\"},\n {type=\"direct\", source=\"entitySnapshot.wizResourceID.id\", target=\"resource.data.wiz_resource_id\"},\n {type=\"direct\", source=\"entitySnapshot.name\", target=\"resource.name\"},\n {type=\"direct\", source=\"entitySnapshot.nativeType\", target=\"resource.type\"},\n {type=\"direct\", source=\"entitySnapshot.providerId\", target=\"resource.uid\"},\n {type=\"direct\", source=\"entitySnapshot.resourceGroupExternalId\", target=\"resource.group.uid\"},\n {type=\"direct\", source=\"entitySnapshot.status\", target=\"resource.status\"},\n {type=\"direct\", source=\"entitySnapshot.type\", target=\"resource.data.generic_type\"},\n \n -- Metadata and event mappings\n {type=\"multi\", source=\"id\", targets={\"metadata.uid\", \"finding.uid\"}},\n {type=\"direct\", source=\"event\", target=\"event.type\"},\n {type=\"direct\", source=\"severity\", target=\"unmapped.severity\"},\n {type=\"multi\", source=\"status\", targets={\"state\", \"status\"}},\n {type=\"multi\", source=\"datasource.vendor\", targets={\"dataSource.vendor\", \"metadata.product.vendor_name\"}},\n {type=\"multi\", source=\"datasource.name\", targets={\"dataSource.name\", \"metadata.product.name\"}},\n {type=\"direct\", source=\"datasource.category\", target=\"dataSource.category\"},\n {type=\"direct\", source=\"ocsf_version\", target=\"metadata.version\"},\n \n -- OCSF standard fields (computed values)\n {type=\"computed\", value=\"2001\", target=\"class_uid\"},\n {type=\"computed\", value=\"99\", target=\"activity_id\"},\n {type=\"computed\", value=\"200199\", target=\"type_uid\"},\n {type=\"computed\", value=\"2\", target=\"category_uid\"},\n {type=\"computed\", value=\"Security Finding\", target=\"class_name\"},\n {type=\"computed\", value=\"Findings\", target=\"category_name\"},\n \n -- Additional required fields\n \n -- Computed mappings using maps\n {type=\"computed_map\", source=\"severity\", map=\"WIZ_SEVERITY_MAP\", default=99, target=\"severity_id\"},\n {type=\"computed_map\", source=\"status\", map=\"WIZ_STATE_MAP\", default=99, target=\"state_id\"},\n \n -- Cloud account type mappings\n {type=\"computed_cloud_type\", source=\"entitySnapshot.cloudPlatform\", target=\"cloud.account.type_id\"},\n {type=\"computed_cloud_type\", source=\"entitySnapshot.cloudPlatform\", target=\"cloud.account.type\"},\n \n}\n\n\n-- Field ordering is now defined by the mapping order in fieldMappings table\n\nfunction getNestedField(obj, path)\n if not obj or not path or path == '' then return nil end\n local current = obj\n for key in string.gmatch(path, '[^.]+') do\n if not current or not key then return nil end\n current = current[key]\n end\n return current\nend\n\nfunction processEvent(event)\n if not event then return event end\n \n -- Populate event.event from _wiz_event_type if it's an issue event\n if event._wiz_event_type and string.lower(event._wiz_event_type) == \"issue\" then\n event.event = \"Issues\"\n end\n \n -- Check if this is an issue event (case-insensitive)\n local isIssueEvent = false\n if event.event and string.lower(event.event) == \"issues\" then\n isIssueEvent = true\n end\n \n if not isIssueEvent then\n -- Return as-is for non-issue events\n return event\n end\n\n -- Add datasource field to event\n if not event.datasource then\n event.datasource = {\n category = \"security\",\n name = \"Wiz\",\n vendor = \"Wiz\"\n }\n end\n \n -- Set event field\n event.event = \"Issues\"\n \n -- Add OCSF fields\n event.class_uid = \"2001\"\n event.activity_id = \"99\"\n event.type_uid = \"200199\"\n event.category_uid = \"2\"\n event.class_name = \"Security Finding\"\n event.category_name = \"Findings\"\n event.analytic_type = \"Rule\"\n event.analytic_type_id = 1\n \n -- Add computed fields \n event.severity_id = WIZ_SEVERITY_MAP[event.severity] or 99\n event.state_id = WIZ_STATE_MAP[event.status] or 99\n \n \n -- Add cloud account type_id (always show) and type (only if cloudPlatform exists)\n local cloudPlatform = nil\n if event.entitySnapshot and event.entitySnapshot.cloudPlatform then\n cloudPlatform = event.entitySnapshot.cloudPlatform\n end\n \n event.cloud_account_type_id = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99\n \n -- Check for AWS IAM users/roles \n if event.cloud_account_type_id == 10 and event.entitySnapshot and event.entitySnapshot.nativeType then\n local nativeType = event.entitySnapshot.nativeType\n if nativeType == \"user\" then\n event.cloud_account_type_id = 3 -- AWS IAM User\n elseif nativeType == \"role\" then\n event.cloud_account_type_id = 4 -- AWS IAM Role\n end\n end\n \n -- Only set cloud_account_type if cloudPlatform exists\n if cloudPlatform then\n event.cloud_account_type = OCSF_ACCOUNT_TYPE_NAME[event.cloud_account_type_id] or cloudPlatform or \"Unknown Account Type\"\n end\n -- If cloudPlatform is nil, don't set cloud_account_type but type_id still shows\n \n -- Add wizResourceID to entitySnapshot \n if event.entitySnapshot and event.entitySnapshot.id then\n event.entitySnapshot.wizResourceID = {\n id = event.entitySnapshot.id\n }\n end\n \n -- Convert timestamps to Unix milliseconds \n if event.createdAt then\n -- Convert createdAt to Unix milliseconds\n local createdAtStr = tostring(event.createdAt)\n -- Try to parse as ISO timestamp and convert to milliseconds\n local success, result = pcall(function()\n -- Simple conversion for ISO timestamps like \"2025-10-08T22:12:42.66996Z\"\n if createdAtStr:match(\"^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d\") then\n -- Extract the timestamp part and convert\n local year, month, day, hour, min, sec = createdAtStr:match(\"^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)\")\n if year and month and day and hour and min and sec then\n -- Create a simple timestamp (this is a basic implementation)\n -- In a real implementation, you'd use proper date parsing\n local timestamp = os.time({\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec)\n })\n return timestamp * 1000 -- Convert to milliseconds\n end\n end\n -- Fallback: try to convert as number\n local num = tonumber(createdAtStr)\n if num then\n -- If it's already a timestamp, ensure it's in milliseconds\n if num > 1000000000000 then -- Already in milliseconds\n return num\n elseif num > 1000000000 then -- In seconds, convert to milliseconds\n return num * 1000\n else -- Assume it's already in the right format\n return num\n end\n end\n return nil\n end)\n \n if success and result then\n event.createdAt = result\n end\n end\n \n if event.updatedAt then\n -- Convert updatedAt to Unix milliseconds\n local updatedAtStr = tostring(event.updatedAt)\n local success, result = pcall(function()\n -- Simple conversion for ISO timestamps\n if updatedAtStr:match(\"^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d\") then\n local year, month, day, hour, min, sec = updatedAtStr:match(\"^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)\")\n if year and month and day and hour and min and sec then\n local timestamp = os.time({\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec)\n })\n return timestamp * 1000\n end\n end\n local num = tonumber(updatedAtStr)\n if num then\n if num > 1000000000000 then\n return num\n elseif num > 1000000000 then\n return num * 1000\n else\n return num\n end\n end\n return nil\n end)\n \n if success and result then\n event.updatedAt = result\n end\n end\n \n -- Set OCSF version \n event.ocsf_version = OCSF_VERSION\n \n -- Remove tags field from entitySnapshot \n if event.entitySnapshot then\n event.entitySnapshot.tags = nil\n end\n\n -- Track all fields that were added to the event as mapped\n -- so they don't appear in unmapped\n local mappedFields = {}\n mappedFields[\"class_uid\"] = true\n mappedFields[\"activity_id\"] = true\n mappedFields[\"type_uid\"] = true\n mappedFields[\"category_uid\"] = true\n mappedFields[\"class_name\"] = true\n mappedFields[\"category_name\"] = true\n mappedFields[\"analytic_type\"] = true\n mappedFields[\"analytic_type_id\"] = true\n mappedFields[\"severity_id\"] = true\n mappedFields[\"state_id\"] = true\n mappedFields[\"cloud_account_type_id\"] = true\n mappedFields[\"cloud_account_type\"] = true\n mappedFields[\"ocsf_version\"] = true\n mappedFields[\"event\"] = true\n mappedFields[\"entitySnapshot\"] = true\n\n local result = {}\n \n -- Track field order as they're processed\n local fieldOrder = {}\n local processedFields = {}\n \n -- Helper functions\n local function getValue(priority1, priority2, priority3)\n local value = getNestedField(event, priority1)\n if value then return value end\n if priority2 then\n value = getNestedField(event, priority2)\n if value then return value end\n end\n if priority3 then\n return getNestedField(event, priority3)\n end\n return nil\n end\n \n local function setValue(targetPath, value)\n if value == \"NULL_PLACEHOLDER\" or value == \"-\" or value == \"\" then\n setNestedField(result, targetPath, nil)\n elseif value ~= nil then\n setNestedField(result, targetPath, value)\n end\n end\n \n local function setMultiValue(targetPaths, value)\n if value == \"NULL_PLACEHOLDER\" or value == \"-\" or value == \"\" then\n for _, targetPath in ipairs(targetPaths) do\n setNestedField(result, targetPath, nil)\n end\n elseif value then\n for _, targetPath in ipairs(targetPaths) do\n setNestedField(result, targetPath, value)\n end\n end\n end\n \n -- Track field order as they're processed\n local fieldOrder = {}\n local processedFields = {}\n \n -- Apply mappings\n for _, mapping in ipairs(fieldMappings) do\n local value = nil\n \n if mapping.type == \"computed\" then\n value = mapping.value\n elseif mapping.type == \"computed_map\" then\n -- Computed value from source using map\n local sourceValue = getNestedField(event, mapping.source)\n local mapTable = nil\n if mapping.map == \"WIZ_SEVERITY_MAP\" then\n mapTable = WIZ_SEVERITY_MAP\n elseif mapping.map == \"WIZ_STATE_MAP\" then\n mapTable = WIZ_STATE_MAP\n end\n value = (mapTable and mapTable[sourceValue]) or mapping.default\n elseif mapping.type == \"computed_cloud_type\" then\n -- Handle cloud account type logic with AWS IAM support\n local cloudPlatform = getNestedField(event, mapping.source)\n local typeId = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99\n \n -- Check for AWS IAM users/roles \n if typeId == 10 and getNestedField(event, \"entitySnapshot.nativeType\") then\n local nativeType = getNestedField(event, \"entitySnapshot.nativeType\")\n if nativeType == \"user\" then\n typeId = 3 -- AWS IAM User\n elseif nativeType == \"role\" then\n typeId = 4 -- AWS IAM Role\n end\n end\n \n if mapping.target == \"cloud.account.type_id\" or mapping.target == \"message.cloud_account_type_id\" then\n value = typeId -- Always show type_id\n elseif mapping.target == \"cloud.account.type\" or mapping.target == \"message.cloud_account_type\" then\n -- Only show type if cloudPlatform exists\n if cloudPlatform then\n value = OCSF_ACCOUNT_TYPE_NAME[typeId] or cloudPlatform or \"Unknown Account Type\"\n end\n -- If cloudPlatform is nil, value remains nil (field will be ignored)\n end\n elseif mapping.type == \"message_field\" then\n value = getNestedField(event, mapping.source)\n elseif mapping.priority1 and mapping.priority2 then\n value = getValue(mapping.priority1, mapping.priority2, mapping.priority3)\n else\n value = getNestedField(event, mapping.source)\n end\n \n -- Set the value and track order\n if mapping.type == \"direct\" then\n setValue(mapping.target, value)\n if not processedFields[mapping.target] then\n table.insert(fieldOrder, mapping.target)\n processedFields[mapping.target] = true\n end\n elseif mapping.type == \"multi\" then\n setMultiValue(mapping.targets, value)\n for _, target in ipairs(mapping.targets) do\n if not processedFields[target] then\n table.insert(fieldOrder, target)\n processedFields[target] = true\n end\n end\n elseif mapping.type == \"computed\" or mapping.type == \"computed_map\" then\n setValue(mapping.target, value)\n if not processedFields[mapping.target] then\n table.insert(fieldOrder, mapping.target)\n processedFields[mapping.target] = true\n end\n elseif mapping.type == \"computed_cloud_type\" then\n if mapping.target:match(\"^message%.\") then\n -- Handle message fields separately\n if not result[\"message\"] then\n result[\"message\"] = {}\n end\n setNestedField(result[\"message\"], mapping.target:gsub(\"message%.\", \"\"), value)\n else\n setValue(mapping.target, value)\n if not processedFields[mapping.target] then\n table.insert(fieldOrder, mapping.target)\n processedFields[mapping.target] = true\n end\n end\n end\n \n -- Track mapped fields for unmapped processing\n if mapping.source then\n mappedFields[mapping.source] = true\n end\n if mapping.priority1 then\n mappedFields[mapping.priority1] = true\n end\n if mapping.priority2 then\n mappedFields[mapping.priority2] = true\n end\n if mapping.priority3 then\n mappedFields[mapping.priority3] = true\n end\n \n -- For computed fields, we need to track the target fields as mapped\n -- since they will be added to the event and shouldn't appear in unmapped\n if mapping.type == \"computed\" or mapping.type == \"computed_map\" or mapping.type == \"computed_cloud_type\" then\n if mapping.target then\n -- Extract the root field name from the target path\n local rootField = mapping.target:match(\"^([^%.]+)\")\n if rootField then\n mappedFields[rootField] = true\n end\n end\n end\n end\n \n -- Only include fields that are NOT mapped and NOT ignored\n for key, value in pairs(event) do\n if not IGNORE_FIELDS[key] and not mappedFields[key] then\n if type(value) == 'table' then\n local nestedObj = {}\n for nestedKey, nestedValue in pairs(value) do\n if type(nestedValue) ~= 'function' and nestedValue ~= \"-\" and nestedValue ~= \"\" then\n -- Check if this nested field was mapped\n local nestedFieldKey = key .. \".\" .. nestedKey\n if not mappedFields[nestedFieldKey] then\n nestedObj[nestedKey] = nestedValue\n end\n end\n end\n if next(nestedObj) then\n result[\"unmapped.\" .. key] = nestedObj\n end\n elseif value ~= \"-\" and value ~= \"\" then\n result[\"unmapped.\" .. key] = value\n end\n end\n end\n \n -- Build nested structure instead of flattening\n local nested = {}\n \n -- Add mapped fields to nested structure\n for _, fieldName in ipairs(fieldOrder) do\n local value = getNestedField(result, fieldName)\n if value ~= nil then \n setNestedField(nested, fieldName, value)\n end\n end\n \n -- Add message field as JSON string (like AWS CloudTrail does)\n local cleanEvent = {}\n for key, value in pairs(event) do\n if key ~= \"_ob\" and key ~= \"timestamp\" and key ~= \"_ts\" and key ~= \"_wiz_event_type\" and key ~= \"_wiz_query_time\" and value ~= \"-\" and value ~= \"\" then\n cleanEvent[key] = value\n end\n end\n \n nested[\"message\"] = encodeJson(cleanEvent, \"message\")\n\n -- Add missing fields that should be in the message\n if cleanEvent.readOnly == nil then\n cleanEvent.readOnly = false -- Default for Wiz events\n end\n if cleanEvent.eventType == nil then\n cleanEvent.eventType = \"Issues\" -- Default for Wiz events\n end\n \n -- Add unmapped fields as nested object (using setNestedField like AWS CloudTrail)\n for key, value in pairs(result) do\n if key:match(\"^unmapped%.\") then\n local fieldName = key:gsub(\"^unmapped%.\", \"\")\n setNestedField(nested, \"unmapped.\" .. fieldName, value)\n end\n end\n\n -- Add time field\n if nested.finding and nested.finding.modified_time then\n nested[\"time\"] = nested.finding.modified_time\n end\n \n if FEATURES.FLATTEN_EVENT_TYPE then\n if nested and nested.event then\n nested['event.type'] = nested.event.type\n end\n end\n return nested\nend\n\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file From acb3a9f903a2f956b54d421d439b196daadfddc1 Mon Sep 17 00:00:00 2001 From: Nate Smalley Date: Sun, 26 Apr 2026 21:41:57 -0700 Subject: [PATCH 2/2] CHANGELOG: 7 broken-legacy transform_ocsf entries removed Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7eaf32..85aace5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,29 @@ The three remaining PAN-OS OCSF transforms in and the field-name convention each expects, so users can choose between them without reading the Lua. No serializer logic changes. +### Removed - 7 broken-legacy `transform_ocsf/` entries + +The following directories have been removed from +`pipelines/community/transform_ocsf/`: + +- `aws_cloudtrail/` +- `aws_guardduty/` +- `darktrace/` +- `gcp_audit_logs/` +- `microsoft_365/` +- `okta/` +- `wiz_issue/` + +Each shares the broken-legacy fingerprint already established by +`palo_alto_networks_firewall/` in the previous release: sub-passing grade +(D or F), `verdict: analyzer_limit`, `class_uid: null`, 0% required-field +coverage, no matching upstream parser in `parsers/community/`, `source_name` +without the `-latest` versioning suffix used by every working entry, and +long-form Python-port style code (632–1720 lines). Each removed entry has +at least one working alternative covering the same vendor cluster +(e.g. `aws_guardduty_logs/`, `darktrace_darktrace_logs/`, `okta_logs/`, +`microsoft_365_mgmt_api_logs/`, `wiz_cloud_security_logs/`). + ## [1.3.0] - 2025-10-28 ### Added