diff --git a/CHANGELOG.md b/CHANGELOG.md index a7eaf32..85aace5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,29 @@ The three remaining PAN-OS OCSF transforms in and the field-name convention each expects, so users can choose between them without reading the Lua. No serializer logic changes. +### Removed - 7 broken-legacy `transform_ocsf/` entries + +The following directories have been removed from +`pipelines/community/transform_ocsf/`: + +- `aws_cloudtrail/` +- `aws_guardduty/` +- `darktrace/` +- `gcp_audit_logs/` +- `microsoft_365/` +- `okta/` +- `wiz_issue/` + +Each shares the broken-legacy fingerprint already established by +`palo_alto_networks_firewall/` in the previous release: sub-passing grade +(D or F), `verdict: analyzer_limit`, `class_uid: null`, 0% required-field +coverage, no matching upstream parser in `parsers/community/`, `source_name` +without the `-latest` versioning suffix used by every working entry, and +long-form Python-port style code (632–1720 lines). Each removed entry has +at least one working alternative covering the same vendor cluster +(e.g. `aws_guardduty_logs/`, `darktrace_darktrace_logs/`, `okta_logs/`, +`microsoft_365_mgmt_api_logs/`, `wiz_cloud_security_logs/`). + ## [1.3.0] - 2025-10-28 ### Added diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/aws_cloudtrail.json b/pipelines/community/transform_ocsf/aws_cloudtrail/aws_cloudtrail.json deleted file mode 100644 index fea2497..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/aws_cloudtrail.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Aws Cloudtrail", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Aws Cloudtrail. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "aws", - "source_name": "aws_cloudtrail", - "version": "1.0.0-rc3", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "aws-cloudtrail-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "\nlocal FEATURES = {\n IGNORE_UNKNOWN_EVENT = true,\n}\n\n-- Field ordering templates for consistent JSON serialization\nlocal FIELD_ORDERS = {\n root = {\"eventVersion\", \"userIdentity\", \"eventTime\", \"eventSource\", \"eventName\", \"awsRegion\", \"sourceIPAddress\", \"userAgent\", \"requestParameters\", \"responseElements\", \"additionalEventData\", \"requestID\", \"eventID\", \"readOnly\", \"resources\", \"eventType\", \"managementEvent\", \"recipientAccountId\", \"sharedEventID\", \"eventCategory\", \"tlsDetails\", \"errorCode\", \"errorMessage\", \"vpcEndpointId\", \"apiVersion\", \"message\", \"time\", \"insightDetails\", \"class_uid\", \"category_uid\", \"metadata\", \"dataSource\"},\n userIdentity = {\"accountId\", \"principalId\", \"accessKeyId\", \"userName\", \"type\", \"invokedBy\", \"sessionContext\"},\n sessionContext = {\"sessionIssuer\", \"attributes\"},\n sessionIssuer = {\"type\", \"principalId\", \"arn\", \"accountId\", \"userName\"},\n attributes = {\"creationDate\"},\n requestParameters = {\"durationSeconds\", \"externalId\", \"bucketName\", \"Host\", \"instanceId\", \"availabilityZone\", \"requestContext\"},\n requestContext = {\"awsAccountId\"},\n responseElements = {\"credentials\"},\n credentials = {\"accessKeyId\", \"expiration\"},\n additionalEventData = {\"SignatureVersion\", \"CipherSuite\", \"bytesTransferredIn\", \"AuthenticationMethod\", \"x-amz-id-2\", \"bytesTransferredOut\"},\n resources = {\"accountId\", \"type\", \"ARN\"},\n tlsDetails = {\"tlsVersion\", \"cipherSuite\"},\n insightDetails = {\"eventSource\", \"insightContext\"},\n insightContext = {\"statistics\"},\n statistics = {\"insightDuration\"},\n metadata = {\"product\"},\n product = {\"name\", \"vendor_name\"},\n dataSource = {\"category\", \"vendor\", \"name\"}\n}\n\n-- Convert ISO 8601 timestamp to Unix epoch milliseconds\nfunction convertToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n \n -- Parse ISO 8601 format: \"2025-09-29T09:15:40Z\" or \"2025-09-29T09:15:40.123Z\"\n local year, month, day, hour, min, sec, ms = string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z\")\n \n if year and month and day and hour and min and sec then\n local t = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n \n -- Get local time interpretation\n local local_seconds = os.time(t)\n \n -- Get what this represents in UTC\n local utc_date = os.date(\"!*t\", local_seconds)\n \n local unix_seconds\n -- Check if the UTC interpretation matches our input\n if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and \n utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and \n utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then\n -- We are already in UTC, use as-is\n unix_seconds = local_seconds\n else\n -- Calculate the correct UTC timestamp\n local utc_seconds = os.time(utc_date)\n local offset = local_seconds - utc_seconds\n unix_seconds = local_seconds + offset -- Add offset to get UTC\n end\n \n -- Add milliseconds if present\n local milli = 0\n if ms and ms ~= \"\" then\n milli = tonumber((ms .. \"000\"):sub(1, 3)) -- pad/truncate to 3 digits\n end\n \n return unix_seconds * 1000 + milli\n end\n \n return nil\nend\n\n-- Optimized JSON encoding function with predefined ordering\nfunction encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" then\n return \"null\"\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n \n if isArray and maxIndex > 0 then\n local items = {}\n for i = 1, maxIndex do\n -- Use the parent key for predefined ordering if available\n local elementKey = key or tostring(i)\n table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or \"null\")\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n local items = {}\n local fieldOrder = FIELD_ORDERS[key] or {}\n \n -- Phase 1: Process fields in predefined order\n for _, fieldName in ipairs(fieldOrder) do\n local v = obj[fieldName]\n if v ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, fieldName))\n end\n end\n \n -- Phase 2: Process remaining fields\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then \n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, keyStr))\n end\n end\n \n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nfunction processEvent(event)\n \n -- Check if eventType is missing and IGNORE_UNKNOWN_EVENT is enabled\n if FEATURES.IGNORE_UNKNOWN_EVENT and (not event.eventType or event.eventType == \"\") then\n return nil\n end\n\n local result = {}\n \n -- Handle responseElements null case\n if event.responseElements == \"\" then\n event.responseElements = nil\n end\n \n -- Direct field mappings for better performance\n local function setField(sourcePath, targetPath)\n local value = getNestedField(event, sourcePath)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n end\n end\n \n -- Priority-based field mapping with fallback\n local function setFieldWithPriority(priority1, priority2, priority3, targetPath)\n local value = getNestedField(event, priority1)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n elseif priority2 then\n value = getNestedField(event, priority2)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n elseif priority3 and priority3 ~= \"\" then\n value = getNestedField(event, priority3)\n if value ~= nil and value ~= \"\" then\n setNestedField(result, targetPath, value)\n end\n end\n end\n end\n \n \n -- Field mapping table for better readability and maintainability\n local fieldMappings = {\n -- Basic CloudTrail fields\n {type = \"direct\", source = \"awsRegion\", target = \"cloud.region\"},\n {type = \"direct\", source = \"eventCategory\", target = \"metadata.product.feature.name\"},\n {type = \"direct\", source = \"eventID\", target = \"metadata.uid\"},\n {type = \"direct\", source = \"eventTime\", target = \"metadata.original_time\"},\n {type = \"direct\", source = \"eventVersion\", target = \"metadata.product.version\"},\n {type = \"direct\", source = \"recipientAccountId\", target = \"cloud.account.uid\"},\n {type = \"priority\", source1 = \"requestID\", source2 = \"requestParameters.externalId\", source3 = \"requestParameters.requestContext.awsAccountId\", target = \"api.request.uid\"},\n {type = \"direct\", source = \"sourceIPAddress\", target = \"src_endpoint.ip\"},\n {type = \"direct\", source = \"userAgent\", target = \"http_request.user_agent\"},\n \n -- User Identity fields\n {type = \"direct\", source = \"userIdentity.principalId\", target = \"actor.user.uid\"},\n {type = \"direct\", source = \"userIdentity.accessKeyId\", target = \"actor.user.credential_uid\"},\n {type = \"direct\", source = \"userIdentity.type\", target = \"actor.user.type\"},\n {type = \"direct\", source = \"userIdentity.invokedBy\", target = \"actor.invoked_by\"},\n {type = \"direct\", source = \"userIdentity.sessionContext.sessionIssuer.principalId\", target = \"actor.session.uid\"},\n {type = \"direct\", source = \"userIdentity.sessionContext.sessionIssuer.userName\", target = \"actor.session.issuer\"},\n {type = \"priority\", source1 = \"userIdentity.userName\", source2 = \"userIdentity.sessionContext.sessionIssuer.type\", target = \"actor.user.name\"},\n {type = \"priority\", source1 = \"userIdentity.accountId\", source2 = \"userIdentity.sessionContext.sessionIssuer.accountId\", target = \"actor.user.account.uid\"},\n \n -- API Information\n {type = \"direct\", source = \"errorCode\", target = \"api.response.error\"},\n {type = \"direct\", source = \"errorMessage\", target = \"api.response.error_message\"},\n \n -- Request Parameters\n {type = \"priority\", source1 = \"requestParameters.durationSeconds\", source2 = \"insightDetails.insightContext.statistics.insightDuration\", target = \"duration\"},\n {type = \"direct\", source = \"requestParameters.bucketName\", target = \"resources.name\"},\n {type = \"direct\", source = \"requestParameters.Host\", target = \"src_endpoint.hostname\"},\n {type = \"direct\", source = \"requestParameters.instanceId\", target = \"src_endpoint.instance_uid\"},\n {type = \"direct\", source = \"requestParameters.availabilityZone\", target = \"cloud.zone\"},\n \n -- Response Elements\n {type = \"direct\", source = \"responseElements.credentials.accessKeyId\", target = \"actor.session.credential_uid\"},\n {type = \"direct\", source = \"responseElements.credentials.expiration\", target = \"actor.session.expiration_time\"},\n \n -- Additional Event Data\n {type = \"direct\", source = \"additionalEventData.x-amz-id-2\", target = \"resources.uid\"},\n {type = \"direct\", source = \"tlsDetails.cipherSuite\", target = \"tls.cipher\"},\n {type = \"direct\", source = \"tlsDetails.tlsVersion\", target = \"tls.version\"},\n \n -- Additional fields\n {type = \"direct\", source = \"vpcEndpointId\", target = \"src_endpoint.uid\"},\n {type = \"direct\", source = \"apiVersion\", target = \"api.version\"},\n {type = \"direct\", source = \"message\", target = \"message\"},\n {type = \"priority\", source1 = \"eventSource\", source2 = \"insightDetails.eventSource\", target = \"api.service.name\"},\n \n -- Individual resource field mappings (for single resource events)\n {type = \"direct\", source = \"resources.accountId\", target = \"resource.account.uid\"},\n {type = \"direct\", source = \"resources.type\", target = \"resource.type\"},\n {type = \"direct\", source = \"resources.ARN\", target = \"resource.uid\"},\n\n -- OCSF field mappings (direct from input)\n {type = \"direct\", source = \"class_uid\", target = \"class_uid\"},\n {type = \"direct\", source = \"category_uid\", target = \"category_uid\"}\n }\n \n -- Process all field mappings in one iteration\n for _, mapping in ipairs(fieldMappings) do\n if mapping.type == \"direct\" then\n setField(mapping.source, mapping.target)\n elseif mapping.type == \"priority\" then\n setFieldWithPriority(mapping.source1, mapping.source2, mapping.source3, mapping.target)\n end\n end\n \n -- Convert timestamps to milliseconds\n local creationDate = getNestedField(event, 'userIdentity.sessionContext.attributes.creationDate')\n if creationDate then\n local convertedTime = convertToMilliseconds(creationDate)\n if convertedTime then\n setNestedField(result, 'actor.session.created_time', convertedTime)\n end\n end\n \n local expirationDate = getNestedField(event, 'responseElements.credentials.expiration')\n if expirationDate then\n local convertedTime = convertToMilliseconds(expirationDate)\n if convertedTime then\n setNestedField(result, 'actor.session.expiration_time', convertedTime)\n end\n end\n \n -- Convert eventTime to time field\n local eventTime = getNestedField(event, 'eventTime')\n if eventTime then\n local convertedTime = convertToMilliseconds(eventTime)\n if convertedTime then\n -- we need to use milliseconds here to be compatible with S1\n setNestedField(result, 'time', convertedTime)\n end\n end\n \n -- Resources array handling\n if event.resources and type(event.resources) == \"table\" then\n local accountIds = {}\n local types = {}\n local arns = {}\n \n for _, resource in ipairs(event.resources) do\n if resource.accountId then table.insert(accountIds, resource.accountId) end\n if resource.type then table.insert(types, resource.type) end\n if resource.ARN then table.insert(arns, resource.ARN) end\n end\n \n if #accountIds > 0 then setNestedField(result, 'resource.account.uid', accountIds) end\n if #types > 0 then setNestedField(result, 'resource.type', types) end\n if #arns > 0 then setNestedField(result, 'resource.uid', arns) end\n end\n \n -- Priority-based default OCSF fields (only set if not already present)\n local function setDefaultIfNotExists(targetPath, defaultValue)\n local existingValue = getNestedField(result, targetPath)\n if existingValue == nil then\n setNestedField(result, targetPath, defaultValue)\n end\n end\n\n -- Static category mapping\n setDefaultIfNotExists('category_uid', 4)\n \n -- Static class mapping \n setDefaultIfNotExists('class_uid', 4002)\n \n -- Set defaults only if fields don't already exist\n setDefaultIfNotExists('metadata.product.name', 'CloudTrail')\n setDefaultIfNotExists('metadata.product.vendor_name', 'AWS')\n setDefaultIfNotExists('metadata.version', '1.0.0-rc3')\n setDefaultIfNotExists('dataSource.vendor', 'AWS')\n setDefaultIfNotExists('dataSource.name', 'CloudTrail')\n setDefaultIfNotExists('dataSource.category', 'security')\n setDefaultIfNotExists('class_name', 'HTTP Activity')\n setDefaultIfNotExists('category_name', 'Network Activity')\n setDefaultIfNotExists('type_name', 'HTTP Activity: Other')\n setDefaultIfNotExists('type_uid', 400299)\n setDefaultIfNotExists('activity_id', 99)\n setDefaultIfNotExists('activity_name', event.eventName or '')\n setDefaultIfNotExists('event.type', event.eventName or '')\n setDefaultIfNotExists('severity_id', 99)\n setDefaultIfNotExists('status_id', 99)\n setDefaultIfNotExists('status', 'Other')\n \n -- Initialize observables array\n local observables = {}\n \n -- Add IP address observable\n if event.sourceIPAddress then\n table.insert(observables, {\n type_id = 2, \n type = 'IP Address', \n name = 'src_endpoint.ip', \n value = event.sourceIPAddress\n })\n end\n \n -- Add ARN observable\n local arn = getNestedField(event, 'userIdentity.arn')\n if arn then\n table.insert(observables, {\n type_id = 99,\n type = 'Other',\n name = 'unmapped.userIdentity.arn',\n value = arn\n })\n end\n \n setNestedField(result, 'observables', observables)\n \n -- Add unmapped fields\n local unmapped = {}\n \n -- Add specific unmapped fields that should be preserved\n if event.eventName then unmapped.eventName = event.eventName end\n if event.readOnly ~= nil then unmapped.readOnly = event.readOnly end\n if event.resources then unmapped.resources = event.resources end\n if event.eventType then unmapped.eventType = event.eventType end\n if event.managementEvent ~= nil then unmapped.managementEvent = event.managementEvent end\n if event.sharedEventID then unmapped.sharedEventID = event.sharedEventID end\n \n -- Add additionalEventData fields as unmapped\n if event.additionalEventData then\n if event.additionalEventData.SignatureVersion then unmapped[\"additionalEventData.SignatureVersion\"] = event.additionalEventData.SignatureVersion end\n if event.additionalEventData.CipherSuite then unmapped[\"additionalEventData.CipherSuite\"] = event.additionalEventData.CipherSuite end\n if event.additionalEventData.bytesTransferredIn then unmapped[\"additionalEventData.bytesTransferredIn\"] = event.additionalEventData.bytesTransferredIn end\n if event.additionalEventData.AuthenticationMethod then unmapped[\"additionalEventData.AuthenticationMethod\"] = event.additionalEventData.AuthenticationMethod end\n if event.additionalEventData.bytesTransferredOut then unmapped[\"additionalEventData.bytesTransferredOut\"] = event.additionalEventData.bytesTransferredOut end\n end\n \n -- Add all unmapped fields automatically\n -- Build comprehensive set of all mapped paths\n local mappedPaths = {}\n \n -- Add computed/mapped-by-logic paths\n mappedPaths['userIdentity.sessionContext.attributes.creationDate'] = true\n mappedPaths['responseElements.credentials.expiration'] = true\n \n -- Add all fieldMappings paths\n for _, mapping in ipairs(fieldMappings) do\n if mapping.type == \"direct\" then\n mappedPaths[mapping.source] = true\n elseif mapping.type == \"priority\" then\n if mapping.source1 then mappedPaths[mapping.source1] = true end\n if mapping.source2 then mappedPaths[mapping.source2] = true end\n if mapping.source3 then mappedPaths[mapping.source3] = true end\n end\n end\n \n for k, v in pairs(event) do\n -- Check if k is used as a source field in mapping\n local is_mapped = mappedPaths[k] == true\n \n -- Filter out Vector-specific fields that shouldn't be in unmapped\n local is_vector_field = (k == \"_ob\" or k == \"site_id\" or k == \"timestamp\")\n \n if not is_mapped and not is_vector_field then\n if type(v) == \"table\" then\n -- For nested objects, filter out only the mapped fields\n local function filterMappedFields(obj, prefix)\n local out = {}\n for nestedKey, nestedValue in pairs(obj) do\n local fullPath = prefix == \"\" and nestedKey or prefix .. \".\" .. nestedKey\n local isNestedMapped = mappedPaths[fullPath] == true\n \n -- Only include unmapped fields\n if not isNestedMapped then\n if type(nestedValue) == \"table\" then\n local child = filterMappedFields(nestedValue, fullPath)\n if next(child) then\n out[nestedKey] = child\n end\n else\n out[nestedKey] = nestedValue\n end\n end\n end\n return out\n end\n \n local filteredObj = filterMappedFields(v, k)\n if next(filteredObj) then\n unmapped[k] = filteredObj\n end\n else\n -- Only add non-null and non-empty values\n if v ~= nil and v ~= \"\" then\n unmapped[k] = v\n end\n end\n end\n end\n \n \n if next(unmapped) then\n setNestedField(result, 'unmapped', unmapped)\n end\n \n -- Create message field with original event\n local cleanEvent = {}\n for key, value in pairs(event) do\n if key ~= \"_ob\" and key ~= \"timestamp\" then\n cleanEvent[key] = value\n end\n end\n \n if event.responseElements == nil then\n cleanEvent.responseElements = \"NULL_PLACEHOLDER\"\n end\n \n -- Add missing fields that should be in the message\n if event.readOnly == nil then\n cleanEvent.readOnly = true -- Default for CloudTrail events\n end\n if event.eventType == nil then\n cleanEvent.eventType = \"AwsApiCall\" -- Default for CloudTrail events\n end\n if event.managementEvent == nil then\n cleanEvent.managementEvent = true -- Default for CloudTrail events\n end\n \n \n -- Flatten result\n local flattened = {}\n flattenObject(result, \"\", flattened)\n flattened.message = encodeJson(cleanEvent, \"root\")\n \n return flattened\nend\n\n-- Simplified helper functions\nfunction setNestedField(obj, path, value)\n if value == nil or path == nil or path == '' then return end\n local keys = {}\n for key in string.gmatch(path, '[^.]+') do\n if key and key ~= '' then\n table.insert(keys, key)\n end\n end\n if #keys == 0 then return end\n \n local current = obj\n for i = 1, #keys - 1 do\n local key = keys[i]\n if current[key] == nil then\n current[key] = {}\n end\n current = current[key]\n end\n current[keys[#keys]] = value\nend\n\nfunction getNestedField(obj, path)\n if obj == nil or path == nil or path == '' then return nil end\n local keys = {}\n for key in string.gmatch(path, '[^.]+') do\n if key and key ~= '' then\n table.insert(keys, key)\n end\n end\n if #keys == 0 then return nil end\n \n local current = obj\n for _, key in ipairs(keys) do\n if current == nil or current[key] == nil then\n return nil\n end\n current = current[key]\n end\n return current\nend\n\nfunction flattenObject(obj, prefix, result)\n prefix = prefix or \"\"\n if type(obj) ~= \"table\" then\n if prefix ~= \"\" then\n result[prefix] = obj\n end\n return\n end\n \n for key, value in pairs(obj) do\n local newKey = prefix == \"\" and key or prefix .. \".\" .. key\n if type(value) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(value) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n if isArray and maxIndex > 0 then\n local arrayValues = {}\n for i = 1, maxIndex do\n if value[i] ~= nil then\n table.insert(arrayValues, value[i])\n end\n end\n result[newKey] = arrayValues\n else\n flattenObject(value, newKey, result)\n end\n else\n result[newKey] = value\n end\n end\nend\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/metadata.yaml b/pipelines/community/transform_ocsf/aws_cloudtrail/metadata.yaml deleted file mode 100644 index f03dbf8..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/metadata.yaml +++ /dev/null @@ -1,52 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Aws Cloudtrail. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: aws - dataSource: Aws Cloudtrail - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "Other - {Explain: object store (S3) with SQS/SNS notifications}" - auth_type: "IAM Role" - sample_record: "{\n \"eventCategory\": \"Management\",\n \"eventName\": \"CreateUser\",\n \"eventSource\"\ - : \"iam.amazonaws.com\",\n \"eventTime\": \"2026-04-20T03:40:52Z\",\n \"eventVersion\": \"1.09\"\ - ,\n \"eventID\": \"4ad68099-cad0-4172-8711-dd15c4d352c9\",\n \"eventType\": \"AwsApiCall\",\n \"\ - awsRegion\": \"us-west-2\",\n \"readOnly\": false,\n \"managementEvent\": true,\n \"recipientAccountId\"\ - : \"987654321098\",\n \"sourceIPAddress\": \"65.40.62.221\",\n \"userAgent\": \"aws-cli/2.15.9 Python/3.11.4\ - \ Linux/5.10\",\n \"tlsDetails\": {\n \"tlsVersion\": \"TLSv1.2\",\n \"cipherSuite\": \"ECDHE-RSA-AES128-GCM-SHA256\"\ - ,\n \"clientProvidedHostHeader\": \"iam.amazonaws.com\"\n },\n \"userIdentity\": {\n \"type\"\ - : \"IAMUser\",\n \"principalId\": \"AIDAMEDICAL6904\",\n \"arn\": \"arn:aws:iam::987654321098:user/karen.martinez\"\ - ,\n \"accountId\": \"987654321098\",\n \"accessKeyId\": \"AKIA2E99153C98AA4B52\",\n \"userName\"\ - : \"karen.martinez\",\n \"sessionContext\": {\n \"sessionIssuer\": {\n \"type\": \"\ - Role\",\n \"principalId\": \"AROANURSE\",\n \"arn\": \"arn:aws:iam::987654321098:role/nurse\"\ - ,\n \"userName\": \"nurse\",\n \"accountId\": \"987654321098\"\n },\n \"attributes\"\ - : {\n \"creationDate\": \"2026-04-20T02:46:52Z\",\n \"mfaAuthenticated\": \"true\"\n\ - \ }\n }\n },\n \"requestID\": \"c32fabb4-8295-49e0-9cb0-8a1833ce49fa\",\n \"requestParameters\"\ - : {\n \"durationSeconds\": 900,\n \"roleArn\": \"arn:aws:iam::987654321098:role/nurse\",\n \ - \ \"roleSessionName\": \"medical-session\",\n \"externalId\": \"a3b57b57-e05c-46ff-9007-2402db4e6004\"\ - ,\n \"userN" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: aws, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/sample.json b/pipelines/community/transform_ocsf/aws_cloudtrail/sample.json deleted file mode 100644 index 4a46ab8..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/sample.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "eventCategory": "Management", - "eventName": "CreateUser", - "eventSource": "iam.amazonaws.com", - "eventTime": "2026-04-20T03:40:52Z", - "eventVersion": "1.09", - "eventID": "4ad68099-cad0-4172-8711-dd15c4d352c9", - "eventType": "AwsApiCall", - "awsRegion": "us-west-2", - "readOnly": false, - "managementEvent": true, - "recipientAccountId": "987654321098", - "sourceIPAddress": "65.40.62.221", - "userAgent": "aws-cli/2.15.9 Python/3.11.4 Linux/5.10", - "tlsDetails": { - "tlsVersion": "TLSv1.2", - "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256", - "clientProvidedHostHeader": "iam.amazonaws.com" - }, - "userIdentity": { - "type": "IAMUser", - "principalId": "AIDAMEDICAL6904", - "arn": "arn:aws:iam::987654321098:user/karen.martinez", - "accountId": "987654321098", - "accessKeyId": "AKIA2E99153C98AA4B52", - "userName": "karen.martinez", - "sessionContext": { - "sessionIssuer": { - "type": "Role", - "principalId": "AROANURSE", - "arn": "arn:aws:iam::987654321098:role/nurse", - "userName": "nurse", - "accountId": "987654321098" - }, - "attributes": { - "creationDate": "2026-04-20T02:46:52Z", - "mfaAuthenticated": "true" - } - } - }, - "requestID": "c32fabb4-8295-49e0-9cb0-8a1833ce49fa", - "requestParameters": { - "durationSeconds": 900, - "roleArn": "arn:aws:iam::987654321098:role/nurse", - "roleSessionName": "medical-session", - "externalId": "a3b57b57-e05c-46ff-9007-2402db4e6004", - "userName": "contractor.johnson", - "tags": [ - { - "Key": "Office", - "Value": "Chicago" - }, - { - "Key": "Department", - "Value": "Science" - } - ] - }, - "responseElements": { - "assumedRoleUser": { - "assumedRoleId": "AROANURSE:medical-session", - "arn": "arn:aws:sts::987654321098:assumed-role/nurse/medical-session" - }, - "credentials": { - "accessKeyId": "ASIADD7D979CACA44C1E", - "sessionToken": "IQoJb3JpZ2luX2VjEJ7//////////wEaCXVzLWVhc3QtMSJHMEUCIQD0b62c804fb42418e83d6fe294790a301", - "expiration": "2026-04-20T04:40:52Z" - }, - "sourceIdentity": "karen.martinez" - }, - "sharedEventID": "3d06df36-b385-486b-969f-f8f427a3f731", - "vpcEndpointId": "vpce-medical-55b991bf5", - "resources": [ - { - "accountId": "987654321098", - "type": "AWS::S3::Bucket", - "ARN": "arn:aws:s3:::backup-sensor-data" - } - ], - "additionalEventData": { - "SignatureVersion": "SigV4", - "CipherSuite": "ECDHE-RSA-AES256-GCM-SHA384", - "bytesTransferredIn": 0, - "bytesTransferredOut": 6607, - "AuthenticationMethod": "AuthHeader", - "x-amz-id-2": "6cd81f33b33044aa9040972beaf35e0e" - }, - "message": "karen.martinez from medical executed CreateUser on iam.amazonaws.com" -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_cloudtrail/serializer.lua b/pipelines/community/transform_ocsf/aws_cloudtrail/serializer.lua deleted file mode 100644 index 47611c6..0000000 --- a/pipelines/community/transform_ocsf/aws_cloudtrail/serializer.lua +++ /dev/null @@ -1,552 +0,0 @@ - -local FEATURES = { - IGNORE_UNKNOWN_EVENT = true, -} - --- Field ordering templates for consistent JSON serialization -local FIELD_ORDERS = { - root = {"eventVersion", "userIdentity", "eventTime", "eventSource", "eventName", "awsRegion", "sourceIPAddress", "userAgent", "requestParameters", "responseElements", "additionalEventData", "requestID", "eventID", "readOnly", "resources", "eventType", "managementEvent", "recipientAccountId", "sharedEventID", "eventCategory", "tlsDetails", "errorCode", "errorMessage", "vpcEndpointId", "apiVersion", "message", "time", "insightDetails", "class_uid", "category_uid", "metadata", "dataSource"}, - userIdentity = {"accountId", "principalId", "accessKeyId", "userName", "type", "invokedBy", "sessionContext"}, - sessionContext = {"sessionIssuer", "attributes"}, - sessionIssuer = {"type", "principalId", "arn", "accountId", "userName"}, - attributes = {"creationDate"}, - requestParameters = {"durationSeconds", "externalId", "bucketName", "Host", "instanceId", "availabilityZone", "requestContext"}, - requestContext = {"awsAccountId"}, - responseElements = {"credentials"}, - credentials = {"accessKeyId", "expiration"}, - additionalEventData = {"SignatureVersion", "CipherSuite", "bytesTransferredIn", "AuthenticationMethod", "x-amz-id-2", "bytesTransferredOut"}, - resources = {"accountId", "type", "ARN"}, - tlsDetails = {"tlsVersion", "cipherSuite"}, - insightDetails = {"eventSource", "insightContext"}, - insightContext = {"statistics"}, - statistics = {"insightDuration"}, - metadata = {"product"}, - product = {"name", "vendor_name"}, - dataSource = {"category", "vendor", "name"} -} - --- Convert ISO 8601 timestamp to Unix epoch milliseconds -function convertToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - - -- Parse ISO 8601 format: "2025-09-29T09:15:40Z" or "2025-09-29T09:15:40.123Z" - local year, month, day, hour, min, sec, ms = string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z") - - if year and month and day and hour and min and sec then - local t = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - - -- Get local time interpretation - local local_seconds = os.time(t) - - -- Get what this represents in UTC - local utc_date = os.date("!*t", local_seconds) - - local unix_seconds - -- Check if the UTC interpretation matches our input - if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and - utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and - utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then - -- We are already in UTC, use as-is - unix_seconds = local_seconds - else - -- Calculate the correct UTC timestamp - local utc_seconds = os.time(utc_date) - local offset = local_seconds - utc_seconds - unix_seconds = local_seconds + offset -- Add offset to get UTC - end - - -- Add milliseconds if present - local milli = 0 - if ms and ms ~= "" then - milli = tonumber((ms .. "000"):sub(1, 3)) -- pad/truncate to 3 digits - end - - return unix_seconds * 1000 + milli - end - - return nil -end - --- Optimized JSON encoding function with predefined ordering -function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray and maxIndex > 0 then - local items = {} - for i = 1, maxIndex do - -- Use the parent key for predefined ordering if available - local elementKey = key or tostring(i) - table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or "null") - end - return "[" .. table.concat(items, ", ") .. "]" - else - local items = {} - local fieldOrder = FIELD_ORDERS[key] or {} - - -- Phase 1: Process fields in predefined order - for _, fieldName in ipairs(fieldOrder) do - local v = obj[fieldName] - if v ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '": ' .. encodeJson(v, fieldName)) - end - end - - -- Phase 2: Process remaining fields - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. encodeJson(v, keyStr)) - end - end - - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -function processEvent(event) - - -- Check if eventType is missing and IGNORE_UNKNOWN_EVENT is enabled - if FEATURES.IGNORE_UNKNOWN_EVENT and (not event.eventType or event.eventType == "") then - return nil - end - - local result = {} - - -- Handle responseElements null case - if event.responseElements == "" then - event.responseElements = nil - end - - -- Direct field mappings for better performance - local function setField(sourcePath, targetPath) - local value = getNestedField(event, sourcePath) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - end - end - - -- Priority-based field mapping with fallback - local function setFieldWithPriority(priority1, priority2, priority3, targetPath) - local value = getNestedField(event, priority1) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - elseif priority2 then - value = getNestedField(event, priority2) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - elseif priority3 and priority3 ~= "" then - value = getNestedField(event, priority3) - if value ~= nil and value ~= "" then - setNestedField(result, targetPath, value) - end - end - end - end - - - -- Field mapping table for better readability and maintainability - local fieldMappings = { - -- Basic CloudTrail fields - {type = "direct", source = "awsRegion", target = "cloud.region"}, - {type = "direct", source = "eventCategory", target = "metadata.product.feature.name"}, - {type = "direct", source = "eventID", target = "metadata.uid"}, - {type = "direct", source = "eventTime", target = "metadata.original_time"}, - {type = "direct", source = "eventVersion", target = "metadata.product.version"}, - {type = "direct", source = "recipientAccountId", target = "cloud.account.uid"}, - {type = "priority", source1 = "requestID", source2 = "requestParameters.externalId", source3 = "requestParameters.requestContext.awsAccountId", target = "api.request.uid"}, - {type = "direct", source = "sourceIPAddress", target = "src_endpoint.ip"}, - {type = "direct", source = "userAgent", target = "http_request.user_agent"}, - - -- User Identity fields - {type = "direct", source = "userIdentity.principalId", target = "actor.user.uid"}, - {type = "direct", source = "userIdentity.accessKeyId", target = "actor.user.credential_uid"}, - {type = "direct", source = "userIdentity.type", target = "actor.user.type"}, - {type = "direct", source = "userIdentity.invokedBy", target = "actor.invoked_by"}, - {type = "direct", source = "userIdentity.sessionContext.sessionIssuer.principalId", target = "actor.session.uid"}, - {type = "direct", source = "userIdentity.sessionContext.sessionIssuer.userName", target = "actor.session.issuer"}, - {type = "priority", source1 = "userIdentity.userName", source2 = "userIdentity.sessionContext.sessionIssuer.type", target = "actor.user.name"}, - {type = "priority", source1 = "userIdentity.accountId", source2 = "userIdentity.sessionContext.sessionIssuer.accountId", target = "actor.user.account.uid"}, - - -- API Information - {type = "direct", source = "errorCode", target = "api.response.error"}, - {type = "direct", source = "errorMessage", target = "api.response.error_message"}, - - -- Request Parameters - {type = "priority", source1 = "requestParameters.durationSeconds", source2 = "insightDetails.insightContext.statistics.insightDuration", target = "duration"}, - {type = "direct", source = "requestParameters.bucketName", target = "resources.name"}, - {type = "direct", source = "requestParameters.Host", target = "src_endpoint.hostname"}, - {type = "direct", source = "requestParameters.instanceId", target = "src_endpoint.instance_uid"}, - {type = "direct", source = "requestParameters.availabilityZone", target = "cloud.zone"}, - - -- Response Elements - {type = "direct", source = "responseElements.credentials.accessKeyId", target = "actor.session.credential_uid"}, - {type = "direct", source = "responseElements.credentials.expiration", target = "actor.session.expiration_time"}, - - -- Additional Event Data - {type = "direct", source = "additionalEventData.x-amz-id-2", target = "resources.uid"}, - {type = "direct", source = "tlsDetails.cipherSuite", target = "tls.cipher"}, - {type = "direct", source = "tlsDetails.tlsVersion", target = "tls.version"}, - - -- Additional fields - {type = "direct", source = "vpcEndpointId", target = "src_endpoint.uid"}, - {type = "direct", source = "apiVersion", target = "api.version"}, - {type = "direct", source = "message", target = "message"}, - {type = "priority", source1 = "eventSource", source2 = "insightDetails.eventSource", target = "api.service.name"}, - - -- Individual resource field mappings (for single resource events) - {type = "direct", source = "resources.accountId", target = "resource.account.uid"}, - {type = "direct", source = "resources.type", target = "resource.type"}, - {type = "direct", source = "resources.ARN", target = "resource.uid"}, - - -- OCSF field mappings (direct from input) - {type = "direct", source = "class_uid", target = "class_uid"}, - {type = "direct", source = "category_uid", target = "category_uid"} - } - - -- Process all field mappings in one iteration - for _, mapping in ipairs(fieldMappings) do - if mapping.type == "direct" then - setField(mapping.source, mapping.target) - elseif mapping.type == "priority" then - setFieldWithPriority(mapping.source1, mapping.source2, mapping.source3, mapping.target) - end - end - - -- Convert timestamps to milliseconds - local creationDate = getNestedField(event, 'userIdentity.sessionContext.attributes.creationDate') - if creationDate then - local convertedTime = convertToMilliseconds(creationDate) - if convertedTime then - setNestedField(result, 'actor.session.created_time', convertedTime) - end - end - - local expirationDate = getNestedField(event, 'responseElements.credentials.expiration') - if expirationDate then - local convertedTime = convertToMilliseconds(expirationDate) - if convertedTime then - setNestedField(result, 'actor.session.expiration_time', convertedTime) - end - end - - -- Convert eventTime to time field - local eventTime = getNestedField(event, 'eventTime') - if eventTime then - local convertedTime = convertToMilliseconds(eventTime) - if convertedTime then - -- we need to use milliseconds here to be compatible with S1 - setNestedField(result, 'time', convertedTime) - end - end - - -- Resources array handling - if event.resources and type(event.resources) == "table" then - local accountIds = {} - local types = {} - local arns = {} - - for _, resource in ipairs(event.resources) do - if resource.accountId then table.insert(accountIds, resource.accountId) end - if resource.type then table.insert(types, resource.type) end - if resource.ARN then table.insert(arns, resource.ARN) end - end - - if #accountIds > 0 then setNestedField(result, 'resource.account.uid', accountIds) end - if #types > 0 then setNestedField(result, 'resource.type', types) end - if #arns > 0 then setNestedField(result, 'resource.uid', arns) end - end - - -- Priority-based default OCSF fields (only set if not already present) - local function setDefaultIfNotExists(targetPath, defaultValue) - local existingValue = getNestedField(result, targetPath) - if existingValue == nil then - setNestedField(result, targetPath, defaultValue) - end - end - - -- Static category mapping - setDefaultIfNotExists('category_uid', 4) - - -- Static class mapping - setDefaultIfNotExists('class_uid', 4002) - - -- Set defaults only if fields don't already exist - setDefaultIfNotExists('metadata.product.name', 'CloudTrail') - setDefaultIfNotExists('metadata.product.vendor_name', 'AWS') - setDefaultIfNotExists('metadata.version', '1.0.0-rc3') - setDefaultIfNotExists('dataSource.vendor', 'AWS') - setDefaultIfNotExists('dataSource.name', 'CloudTrail') - setDefaultIfNotExists('dataSource.category', 'security') - setDefaultIfNotExists('class_name', 'HTTP Activity') - setDefaultIfNotExists('category_name', 'Network Activity') - setDefaultIfNotExists('type_name', 'HTTP Activity: Other') - setDefaultIfNotExists('type_uid', 400299) - setDefaultIfNotExists('activity_id', 99) - setDefaultIfNotExists('activity_name', event.eventName or '') - setDefaultIfNotExists('event.type', event.eventName or '') - setDefaultIfNotExists('severity_id', 99) - setDefaultIfNotExists('status_id', 99) - setDefaultIfNotExists('status', 'Other') - - -- Initialize observables array - local observables = {} - - -- Add IP address observable - if event.sourceIPAddress then - table.insert(observables, { - type_id = 2, - type = 'IP Address', - name = 'src_endpoint.ip', - value = event.sourceIPAddress - }) - end - - -- Add ARN observable - local arn = getNestedField(event, 'userIdentity.arn') - if arn then - table.insert(observables, { - type_id = 99, - type = 'Other', - name = 'unmapped.userIdentity.arn', - value = arn - }) - end - - setNestedField(result, 'observables', observables) - - -- Add unmapped fields - local unmapped = {} - - -- Add specific unmapped fields that should be preserved - if event.eventName then unmapped.eventName = event.eventName end - if event.readOnly ~= nil then unmapped.readOnly = event.readOnly end - if event.resources then unmapped.resources = event.resources end - if event.eventType then unmapped.eventType = event.eventType end - if event.managementEvent ~= nil then unmapped.managementEvent = event.managementEvent end - if event.sharedEventID then unmapped.sharedEventID = event.sharedEventID end - - -- Add additionalEventData fields as unmapped - if event.additionalEventData then - if event.additionalEventData.SignatureVersion then unmapped["additionalEventData.SignatureVersion"] = event.additionalEventData.SignatureVersion end - if event.additionalEventData.CipherSuite then unmapped["additionalEventData.CipherSuite"] = event.additionalEventData.CipherSuite end - if event.additionalEventData.bytesTransferredIn then unmapped["additionalEventData.bytesTransferredIn"] = event.additionalEventData.bytesTransferredIn end - if event.additionalEventData.AuthenticationMethod then unmapped["additionalEventData.AuthenticationMethod"] = event.additionalEventData.AuthenticationMethod end - if event.additionalEventData.bytesTransferredOut then unmapped["additionalEventData.bytesTransferredOut"] = event.additionalEventData.bytesTransferredOut end - end - - -- Add all unmapped fields automatically - -- Build comprehensive set of all mapped paths - local mappedPaths = {} - - -- Add computed/mapped-by-logic paths - mappedPaths['userIdentity.sessionContext.attributes.creationDate'] = true - mappedPaths['responseElements.credentials.expiration'] = true - - -- Add all fieldMappings paths - for _, mapping in ipairs(fieldMappings) do - if mapping.type == "direct" then - mappedPaths[mapping.source] = true - elseif mapping.type == "priority" then - if mapping.source1 then mappedPaths[mapping.source1] = true end - if mapping.source2 then mappedPaths[mapping.source2] = true end - if mapping.source3 then mappedPaths[mapping.source3] = true end - end - end - - for k, v in pairs(event) do - -- Check if k is used as a source field in mapping - local is_mapped = mappedPaths[k] == true - - -- Filter out Vector-specific fields that shouldn't be in unmapped - local is_vector_field = (k == "_ob" or k == "site_id" or k == "timestamp") - - if not is_mapped and not is_vector_field then - if type(v) == "table" then - -- For nested objects, filter out only the mapped fields - local function filterMappedFields(obj, prefix) - local out = {} - for nestedKey, nestedValue in pairs(obj) do - local fullPath = prefix == "" and nestedKey or prefix .. "." .. nestedKey - local isNestedMapped = mappedPaths[fullPath] == true - - -- Only include unmapped fields - if not isNestedMapped then - if type(nestedValue) == "table" then - local child = filterMappedFields(nestedValue, fullPath) - if next(child) then - out[nestedKey] = child - end - else - out[nestedKey] = nestedValue - end - end - end - return out - end - - local filteredObj = filterMappedFields(v, k) - if next(filteredObj) then - unmapped[k] = filteredObj - end - else - -- Only add non-null and non-empty values - if v ~= nil and v ~= "" then - unmapped[k] = v - end - end - end - end - - - if next(unmapped) then - setNestedField(result, 'unmapped', unmapped) - end - - -- Create message field with original event - local cleanEvent = {} - for key, value in pairs(event) do - if key ~= "_ob" and key ~= "timestamp" then - cleanEvent[key] = value - end - end - - if event.responseElements == nil then - cleanEvent.responseElements = "NULL_PLACEHOLDER" - end - - -- Add missing fields that should be in the message - if event.readOnly == nil then - cleanEvent.readOnly = true -- Default for CloudTrail events - end - if event.eventType == nil then - cleanEvent.eventType = "AwsApiCall" -- Default for CloudTrail events - end - if event.managementEvent == nil then - cleanEvent.managementEvent = true -- Default for CloudTrail events - end - - - -- Flatten result - local flattened = {} - flattenObject(result, "", flattened) - flattened.message = encodeJson(cleanEvent, "root") - - return flattened -end - --- Simplified helper functions -function setNestedField(obj, path, value) - if value == nil or path == nil or path == '' then return end - local keys = {} - for key in string.gmatch(path, '[^.]+') do - if key and key ~= '' then - table.insert(keys, key) - end - end - if #keys == 0 then return end - - local current = obj - for i = 1, #keys - 1 do - local key = keys[i] - if current[key] == nil then - current[key] = {} - end - current = current[key] - end - current[keys[#keys]] = value -end - -function getNestedField(obj, path) - if obj == nil or path == nil or path == '' then return nil end - local keys = {} - for key in string.gmatch(path, '[^.]+') do - if key and key ~= '' then - table.insert(keys, key) - end - end - if #keys == 0 then return nil end - - local current = obj - for _, key in ipairs(keys) do - if current == nil or current[key] == nil then - return nil - end - current = current[key] - end - return current -end - -function flattenObject(obj, prefix, result) - prefix = prefix or "" - if type(obj) ~= "table" then - if prefix ~= "" then - result[prefix] = obj - end - return - end - - for key, value in pairs(obj) do - local newKey = prefix == "" and key or prefix .. "." .. key - if type(value) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(value) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray and maxIndex > 0 then - local arrayValues = {} - for i = 1, maxIndex do - if value[i] ~= nil then - table.insert(arrayValues, value[i]) - end - end - result[newKey] = arrayValues - else - flattenObject(value, newKey, result) - end - else - result[newKey] = value - end - end -end diff --git a/pipelines/community/transform_ocsf/aws_guardduty/aws_guardduty.json b/pipelines/community/transform_ocsf/aws_guardduty/aws_guardduty.json deleted file mode 100644 index b5701ba..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/aws_guardduty.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Aws Guardduty", - "grade": { - "letter": "F", - "score": 45, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Aws Guardduty. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "aws", - "source_name": "aws_guardduty", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "aws-guardduty-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- AWS GuardDuty OCSF 1.0.0 parser (ported from Python)\n\n-- Helper: split string by delimiter\nlocal function split(str, delimiter)\n local result = {}\n local escaped = delimiter:gsub(\"[%.%+%*%?%^%$%(%)%[%]%%]\", \"%%%1\")\n local pattern = \"([^\" .. escaped .. \"]+)\"\n for token in tostring(str):gmatch(pattern) do\n table.insert(result, token)\n end\n if #result == 0 and #tostring(str) > 0 then\n table.insert(result, str)\n end\n return result\nend\n\n-- Helper: safely access nested keys: keys is array, obj is table\nlocal function getByPath(obj, keys)\n local current = obj\n for _, k in ipairs(keys) do\n if current ~= nil and type(current) == \"table\" then\n current = current[k]\n else\n return nil\n end\n end\n return current\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n\t return value\n end\n local copy = {}\n for k, v in pairs(value) do\n\t if not (ignoreKeys and ignoreKeys[k]) then\n\t\t copy[k] = deepCopy(v, ignoreKeys)\n\t end\n end\n return copy\nend\n\n-- Common mapping applied before specific finders\nlocal function common_mapping(event, site_id)\n event[\"dataSource\"] = { category = \"security\", name = \"AWS GuardDuty\", vendor = \"AWS\" }\n event[\"metadata\"] = { version = \"1.0.0\", product = { name = \"AWS GuardDuty\", vendor_name = \"AWS\" } }\n event[\"cloudProvider\"] = \"AWS\"\n event[\"cloudAccountType\"] = \"AWS Account\"\n event[\"accountTypeId\"] = 10\n if site_id ~= nil and site_id ~= \"\" then\n event[\"site\"] = { id = site_id }\n end\n local sev = event[\"severity\"]\n if type(sev) == \"number\" then\n if sev >= 1.0 and sev <= 2.9 then\n event[\"severity_id\"] = 2\n elseif sev >= 4.0 and sev <= 6.9 then\n event[\"severity_id\"] = 3\n elseif sev >= 7.0 and sev <= 8.9 then\n event[\"severity_id\"] = 4\n else\n event[\"severity_id\"] = 0\n end\n end\nend\n\n-- After mapping, convert lat/lon into coordinates and remove lat/lon fields based on activity_name\nlocal function coordinates_mapping(parsed)\n local lat = parsed[\"lat\"]\n local lon = parsed[\"lon\"]\n if (type(lat) == \"number\" or type(lat) == \"string\") or (type(lon) == \"number\" or type(lon) == \"string\") then\n local latnum = tonumber(lat)\n local lonnum = tonumber(lon)\n if latnum and lonnum then\n local coords = { latnum, lonnum }\n local activity = parsed[\"activity_name\"]\n if activity == \"EC2 finding types\" or\n activity == \"EKS Runtime Monitoring finding types\" or\n activity == \"Kubernetes Audit Logs finding types\" or\n activity == \"RDS Protection finding types\" or\n activity == \"S3 finding types\" then\n parsed[\"src_endpoint.location.coordinates\"] = coords\n parsed[\"lat\"], parsed[\"lon\"] = nil, nil\n elseif activity == \"IAM finding types\" or activity == \"Lambda Protection finding types\" then\n parsed[\"device.location.coordinates\"] = coords\n parsed[\"lat\"], parsed[\"lon\"] = nil, nil\n end\n end\n end\nend\n\nlocal function flatten_table(tbl, prefix, out)\n out = out or {}\n prefix = prefix or \"\"\n local hasArrayElements = false\n \n for k, v in pairs(tbl) do\n if type(k) == \"number\" then\n hasArrayElements = true\n break\n end\n end\n \n if hasArrayElements then\n -- This is an array, store it as-is\n if prefix ~= \"\" then\n out[prefix] = tbl\n end\n return out\n end\n \n for k, v in pairs(tbl) do\n local key = prefix ~= \"\" and (prefix .. \".\" .. k) or k\n if type(v) == \"table\" then\n flatten_table(v, key, out)\n else\n out[key] = v\n end\n end\n \n return out\nend\n\n-- Apply a mapping table of { [source_dotted] = target_dotted }\nlocal function apply_mapping(event, mapping)\n local out = {}\n for src, dst in pairs(mapping) do\n local val = getByPath(event, split(src, \".\"))\n if val ~= nil then\n out[dst] = val\n end\n end\n\n -- NEW: auto-capture unmapped fields\n local copy = deepCopy(event)\n local flattenCopy = flatten_table(copy)\n for key, val in pairs(flattenCopy) do\n if mapping[key] == nil and val ~= nil and val ~= \"\" and val ~= \"[]\" and val ~= \"()\" and val ~= \"{}\" then\n out[\"unmapped.\" .. key] = val\n end\n end\n return out\nend\n\nlocal EC2_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\",\"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"instanceDetails\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\", \n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\",\n \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"networkConnectionAction\"\n },\n networkConnectionAction = {\n \"connectionDirection\", \"localIpDetails\", \"remoteIpDetails\", \"remotePortDetails\", \n \"localPortDetails\", \"protocol\", \"blocked\"\n },\n localIpDetails = {\n \"ipAddressV4\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n remotePortDetails = {\n \"port\", \"portName\"\n },\n localPortDetails = {\n \"port\", \"portName\"\n },\n additionalInfo = {\n \"threatListName\", \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal EKS_RUNTIME_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"eksClusterDetails\", \"kubernetesDetails\", \"containerDetails\", \"instanceDetails\"\n },\n eksClusterDetails = {\n \"name\", \"arn\", \"createdAt\", \"vpcId\", \"status\", \"tags\"\n },\n tags = {\n \"key\", \"value\"\n },\n kubernetesDetails = {\n \"kubernetesWorkloadDetails\"\n },\n kubernetesWorkloadDetails = {\n \"name\", \"namespace\", \"type\", \"uid\"\n },\n containerDetails = {\n \"id\", \"name\", \"image\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"runtimeDetails\", \"featureName\", \"resourceRole\",\n \"additionalInfo\", \"evidence\", \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"dnsRequestAction\"\n },\n dnsRequestAction = {\n \"domain\", \"protocol\", \"blocked\", \"domainWithSuffix\"\n },\n runtimeDetails = {\n \"process\"\n },\n process = {\n \"pid\", \"name\", \"uuid\", \"executablePath\", \"executableSha256\", \"cmdLine\", \"user\", \"euid\",\n \"userId\", \"pwd\", \"startTime\", \"parentUuid\", \"lineage\"\n },\n lineage = {\n \"pid\", \"uuid\", \"executablePath\", \"euid\", \"parentUuid\"\n },\n additionalInfo = {\n \"threatListName\", \"sample\", \"agentDetails\", \"value\", \"type\"\n },\n agentDetails = {\n \"agentVersion\", \"agentId\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal IAM_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"accessKeyDetails\", \"instanceDetails\"\n },\n accessKeyDetails = {\n \"accessKeyId\", \"principalId\", \"userType\", \"userName\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"awsApiCallAction\"\n },\n awsApiCallAction = {\n \"api\", \"serviceName\", \"callerType\", \"errorCode\", \"remoteIpDetails\", \"affectedResources\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"userAgent\", \"anomalies\", \"profiledBehavior\", \"unusualBehavior\", \"sample\", \"value\", \"type\"\n },\n userAgent = {\n \"fullUserAgent\", \"userAgentCategory\"\n },\n anomalies = {\n \"anomalousAPIs\"\n },\n profiledBehavior = {\n \"rareProfiledAPIsAccountProfiling\", \"infrequentProfiledAPIsAccountProfiling\",\n \"frequentProfiledAPIsAccountProfiling\", \"rareProfiledAPIsUserIdentityProfiling\",\n \"infrequentProfiledAPIsUserIdentityProfiling\", \"frequentProfiledAPIsUserIdentityProfiling\",\n \"rareProfiledUserTypesAccountProfiling\", \"infrequentProfiledUserTypesAccountProfiling\",\n \"frequentProfiledUserTypesAccountProfiling\", \"rareProfiledUserNamesAccountProfiling\",\n \"infrequentProfiledUserNamesAccountProfiling\", \"frequentProfiledUserNamesAccountProfiling\",\n \"rareProfiledASNsAccountProfiling\", \"infrequentProfiledASNsAccountProfiling\",\n \"frequentProfiledASNsAccountProfiling\", \"rareProfiledASNsUserIdentityProfiling\",\n \"infrequentProfiledASNsUserIdentityProfiling\", \"frequentProfiledASNsUserIdentityProfiling\",\n \"rareProfiledUserAgentsAccountProfiling\", \"infrequentProfiledUserAgentsAccountProfiling\",\n \"frequentProfiledUserAgentsAccountProfiling\", \"rareProfiledUserAgentsUserIdentityProfiling\",\n \"infrequentProfiledUserAgentsUserIdentityProfiling\", \"frequentProfiledUserAgentsUserIdentityProfiling\"\n },\n unusualBehavior = {\n \"unusualAPIsAccountProfiling\", \"unusualAPIsUserIdentityProfiling\",\n \"unusualUserTypesAccountProfiling\", \"unusualUserNamesAccountProfiling\",\n \"unusualASNsAccountProfiling\", \"unusualASNsUserIdentityProfiling\",\n \"unusualUserAgentsAccountProfiling\", \"unusualUserAgentsUserIdentityProfiling\",\n \"isUnusualUserIdentity\"\n },\n}\n\nlocal KUBERNETES_AUDIT_LOGS_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"eksClusterDetails\", \"kubernetesDetails\", \"accessKeyDetails\"\n },\n eksClusterDetails = {\n \"name\", \"arn\", \"createdAt\", \"vpcId\", \"status\", \"tags\"\n },\n tags = {\n \"key\", \"value\"\n },\n kubernetesDetails = {\n \"kubernetesWorkloadDetails\", \"kubernetesUserDetails\"\n },\n kubernetesWorkloadDetails = {\n \"name\", \"namespace\", \"type\", \"uid\"\n },\n kubernetesUserDetails = {\n \"username\", \"uid\", \"groups\", \"sessionName\"\n },\n accessKeyDetails = {\n \"accessKeyId\", \"principalId\", \"userType\", \"userName\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"kubernetesApiCallAction\"\n },\n kubernetesApiCallAction = {\n \"requestUri\", \"verb\", \"sourceIPs\", \"userAgent\", \"remoteIpDetails\", \"statusCode\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"threatName\", \"threatListName\", \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal LAMBDA_PROTECTION_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"lambdaDetails\"\n },\n lambdaDetails = {\n \"functionArn\", \"functionName\", \"description\", \"lastModifiedAt\", \"revisionId\",\n \"functionVersion\", \"role\", \"vpcConfig\", \"tags\"\n },\n vpcConfig = {\n \"vpcId\", \"securityGroups\", \"subnetIds\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"networkConnectionAction\"\n },\n networkConnectionAction = {\n \"connectionDirection\", \"remoteIpDetails\", \"remotePortDetails\", \"protocol\", \"blocked\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n remotePortDetails = {\n \"port\", \"portName\"\n },\n additionalInfo = {\n \"unusualProtocol\", \"threatListName\", \"unusual\", \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal MALWARE_PROTECTION_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"instanceDetails\", \"ebsVolumeDetails\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n tags = {\n \"key\", \"value\"\n },\n ebsVolumeDetails = {\n \"scannedVolumeDetails\", \"skippedVolumeDetails\"\n },\n scannedVolumeDetails = {\n \"volumeArn\", \"volumeType\", \"deviceName\", \"volumeSizeInGB\", \"encryptionType\", \"snapshotArn\", \"kmsKeyArn\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"featureName\", \"ebsVolumeScanDetails\", \"additionalInfo\", \"evidence\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n ebsVolumeScanDetails = {\n \"scanId\", \"scanStartedAt\", \"scanCompletedAt\", \"scanType\", \"triggerFindingId\", \"sources\", \"scanDetections\"\n },\n scanDetections = {\n \"scannedItemCount\", \"threatsDetectedItemCount\", \"highestSeverityThreatDetails\", \"threatDetectedByName\"\n },\n scannedItemCount = {\n \"totalGb\", \"files\", \"volumes\"\n },\n threatsDetectedItemCount = {\n \"files\"\n },\n highestSeverityThreatDetails = {\n \"severity\", \"threatName\", \"count\"\n },\n threatDetectedByName = {\n \"itemCount\", \"uniqueThreatNameCount\", \"shortened\", \"threatNames\"\n },\n threatNames = {\n \"name\", \"severity\", \"itemCount\", \"filePaths\"\n },\n filePaths = {\n \"filePath\", \"fileName\", \"volumeArn\", \"hash\"\n },\n additionalInfo = {\n \"sample\", \"value\", \"type\"\n },\n}\n\nlocal RDS_PROTECTION_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"rdsDbInstanceDetails\", \"rdsDbUserDetails\"\n },\n rdsDbInstanceDetails = {\n \"dbInstanceIdentifier\", \"engine\", \"engineVersion\", \"dbClusterIdentifier\", \"dbInstanceArn\", \"tags\"\n },\n tags = {\n \"key\", \"value\"\n },\n rdsDbUserDetails = {\n \"user\", \"application\", \"database\", \"ssl\", \"authMethod\"\n },\n service = {\n \"action\", \"additionalInfo\", \"resourceRole\", \"evidence\", \"count\", \"detectorId\",\n \"eventFirstSeen\", \"eventLastSeen\", \"serviceName\", \"archived\"\n },\n action = {\n \"actionType\", \"rdsLoginAttemptAction\"\n },\n rdsLoginAttemptAction = {\n \"remoteIpDetails\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"sample\", \"value\", \"type\"\n },\n evidence = {\n \"threatIntelligenceDetails\"\n },\n threatIntelligenceDetails = {\n \"threatListName\", \"threatNames\"\n },\n}\n\nlocal S3_FIELD_ORDER = {\n message = {\n \"schemaVersion\", \"accountId\", \"region\", \"partition\", \"id\", \"arn\", \"type\", \"resource\",\n \"service\", \"severity\", \"createdAt\", \"updatedAt\", \"title\", \"description\"\n },\n resource = {\n \"resourceType\", \"accessKeyDetails\", \"s3BucketDetails\", \"instanceDetails\"\n },\n accessKeyDetails = {\n \"accessKeyId\", \"principalId\", \"userType\", \"userName\"\n },\n s3BucketDetails = {\n \"arn\", \"name\", \"type\", \"createdAt\", \"owner\", \"tags\", \"defaultServerSideEncryption\", \"publicAccess\"\n },\n owner = {\n \"id\"\n },\n tags = {\n \"key\", \"value\"\n },\n defaultServerSideEncryption = {\n \"encryptionType\", \"kmsMasterKeyArn\"\n },\n publicAccess = {\n \"permissionConfiguration\", \"effectivePermission\"\n },\n permissionConfiguration = {\n \"bucketLevelPermissions\", \"accountLevelPermissions\"\n },\n bucketLevelPermissions = {\n \"accessControlList\", \"bucketPolicy\", \"blockPublicAccess\"\n },\n accessControlList = {\n \"allowsPublicReadAccess\", \"allowsPublicWriteAccess\"\n },\n bucketPolicy = {\n \"allowsPublicReadAccess\", \"allowsPublicWriteAccess\"\n },\n blockPublicAccess = {\n \"ignorePublicAcls\", \"restrictPublicBuckets\", \"blockPublicAcls\", \"blockPublicPolicy\"\n },\n accountLevelPermissions = {\n \"blockPublicAccess\"\n },\n instanceDetails = {\n \"instanceId\", \"instanceType\", \"outpostArn\", \"launchTime\", \"platform\", \"productCodes\",\n \"iamInstanceProfile\", \"networkInterfaces\", \"tags\", \"instanceState\", \"availabilityZone\",\n \"imageId\", \"imageDescription\"\n },\n productCodes = {\n \"productCodeId\", \"productCodeType\"\n },\n iamInstanceProfile = {\n \"arn\", \"id\"\n },\n networkInterfaces = {\n \"ipv6Addresses\", \"networkInterfaceId\", \"privateDnsName\", \"privateIpAddress\",\n \"privateIpAddresses\", \"subnetId\", \"vpcId\", \"securityGroups\", \"publicDnsName\", \"publicIp\"\n },\n privateIpAddresses = {\n \"privateDnsName\", \"privateIpAddress\"\n },\n securityGroups = {\n \"groupName\", \"groupId\"\n },\n service = {\n \"serviceName\", \"detectorId\", \"action\", \"resourceRole\", \"additionalInfo\",\n \"eventFirstSeen\", \"eventLastSeen\", \"archived\", \"count\"\n },\n action = {\n \"actionType\", \"awsApiCallAction\"\n },\n awsApiCallAction = {\n \"api\", \"serviceName\", \"callerType\", \"errorCode\", \"remoteIpDetails\", \"affectedResources\"\n },\n remoteIpDetails = {\n \"ipAddressV4\", \"organization\", \"country\", \"city\", \"geoLocation\"\n },\n organization = {\n \"asn\", \"asnOrg\", \"isp\", \"org\"\n },\n country = {\n \"countryName\"\n },\n city = {\n \"cityName\"\n },\n geoLocation = {\n \"lat\", \"lon\"\n },\n additionalInfo = {\n \"unusual\", \"sample\", \"value\", \"type\"\n },\n unusual = {\n \"hoursOfDay\", \"userNames\"\n },\n}\n\nlocal function get_msg_field_ordering(family)\n if family == \"ec2\" then return EC2_FIELD_ORDER end\n if family == \"runtime\" or family == \"eks_runtime\" or family == \"runtime_monitoring\" then return EKS_RUNTIME_FIELD_ORDER end\n if family == \"iam\" then return IAM_FIELD_ORDER end\n if family == \"kubernetes\" or family == \"k8s\" then return KUBERNETES_AUDIT_LOGS_FIELD_ORDER end\n if family == \"lambda\" then return LAMBDA_PROTECTION_FIELD_ORDER end\n if family == \"malware\" then return MALWARE_PROTECTION_FIELD_ORDER end\n if family == \"rds\" then return RDS_PROTECTION_FIELD_ORDER end\n if family == \"s3\" then return S3_FIELD_ORDER end\n return {}\nend\n\n-- OCSF Mappings (ported 1:1 from Python)\nlocal function ec2_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.instanceDetails.instanceId\"] = \"device.instance_uid\",\n [\"resource.instanceDetails.networkInterfaces\"] = \"device.network_interfaces\",\n [\"resource.instanceDetails.imageId\"] = \"device.image.uid\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.action.networkConnectionAction.localIpDetails.ipAddressV4\"] = \"src_endpoint.ip\",\n [\"service.action.networkConnectionAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.networkConnectionAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.networkConnectionAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.action.networkConnectionAction.localPortDetails.port\"] = \"src_endpoint.port\",\n [\"service.action.networkConnectionAction.protocol\"] = \"connection_info.protocol_name\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function runtime_monitoring_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.instanceDetails.instanceId\"] = \"device.instance_uid\",\n [\"resource.instanceDetails.networkInterfaces\"] = \"device.network_interfaces\",\n [\"resource.instanceDetails.imageId\"] = \"device.image.uid\",\n [\"service.runtimeDetails.process.pid\"] = \"actor.process.uid\",\n [\"service.runtimeDetails.process.name\"] = \"actor.process.name\",\n [\"service.runtimeDetails.process.user\"] = \"actor.user.name\",\n [\"service.runtimeDetails.process.userId\"] = \"actor.user.uid\",\n [\"service.runtimeDetails.process.parentUuid\"] = \"actor.process.parent_process.uid\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.featureName\"] = \"api.service.name\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function iam_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.accessKeyDetails.accessKeyId\"] = \"actor.session.credential_uid\",\n [\"resource.accessKeyDetails.principalId\"] = \"actor.session.uid\",\n [\"resource.accessKeyDetails.userName\"] = \"actor.user.name\",\n [\"resource.instanceDetails.instanceId\"] = \"device.instance_uid\",\n [\"resource.instanceDetails.networkInterfaces\"] = \"device.network_interfaces\",\n [\"resource.instanceDetails.imageId\"] = \"device.image.uid\",\n [\"service.action.awsApiCallAction.serviceName\"] = \"api.service.name\",\n [\"service.action.awsApiCallAction.remoteIpDetails.organization.isp\"] = \"device.location.isp\",\n [\"service.action.awsApiCallAction.remoteIpDetails.country.countryName\"] = \"device.location.country\",\n [\"service.action.awsApiCallAction.remoteIpDetails.city.cityName\"] = \"device.location.city\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function kubernetes_audit_log_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.kubernetesDetails.kubernetesUserDetails.username\"] = \"actor.user.name\",\n [\"resource.kubernetesDetails.kubernetesUserDetails.uid\"] = \"actor.user.uid\",\n [\"resource.kubernetesDetails.kubernetesUserDetails.groups\"] = \"actor.user.groups\",\n [\"resource.accessKeyDetails.accessKeyId\"] = \"actor.session.credential_uid\",\n [\"resource.accessKeyDetails.principalId\"] = \"actor.session.uid\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.action.kubernetesApiCallAction.requestUri\"] = \"http_request.url.url_string\",\n [\"service.action.kubernetesApiCallAction.verb\"] = \"http_request.http_method\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.action.kubernetesApiCallAction.statusCode\"] = \"status_code\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function lambda_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"service.action.networkConnectionAction.remoteIpDetails.organization.isp\"] = \"device.location.isp\",\n [\"service.action.networkConnectionAction.remoteIpDetails.country.countryName\"] = \"device.location.country\",\n [\"service.action.networkConnectionAction.remoteIpDetails.city.cityName\"] = \"device.location.city\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function malware_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"partition\"] = \"resources.cloud_partition\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"service.featureName\"] = \"api.service.name\",\n [\"service.ebsVolumeScanDetails.scanId\"] = \"api.service.uid\",\n [\"service.ebsVolumeScanDetails.scanStartedAt\"] = \"finding.first_seen_time\",\n [\"service.ebsVolumeScanDetails.scanCompletedAt\"] = \"finding.last_seen_time\",\n [\"service.ebsVolumeScanDetails.triggerFindingId\"] = \"finding.uid\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function rds_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function s3_findings_ocsf_mapping()\n return {\n [\"schemaVersion\"] = \"metadata.log_version\",\n [\"accountId\"] = \"cloud.account.uid\",\n [\"region\"] = \"cloud.region\",\n [\"partition\"] = \"resources.cloud_partition\",\n [\"id\"] = \"metadata.uid\",\n [\"type\"] = \"metadata.log_name\",\n [\"resource.resourceType\"] = \"metadata.log_provider\",\n [\"resource.accessKeyDetails.accessKeyId\"] = \"resources.user.credential_uid\",\n [\"resource.accessKeyDetails.userType\"] = \"resources.user.type\",\n [\"resource.accessKeyDetails.userName\"] = \"resources.user.name\",\n [\"service.serviceName\"] = \"src_endpoint.svc_name\",\n [\"service.detectorId\"] = \"src_endpoint.uid\",\n [\"service.action.awsApiCallAction.serviceName\"] = \"api.service.name\",\n [\"service.action.awsApiCallAction.remoteIpDetails.organization.isp\"] = \"src_endpoint.location.isp\",\n [\"service.action.awsApiCallAction.remoteIpDetails.country.countryName\"] = \"src_endpoint.location.country\",\n [\"service.action.awsApiCallAction.remoteIpDetails.city.cityName\"] = \"src_endpoint.location.city\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat\"] = \"lat\",\n [\"service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon\"] = \"lon\",\n [\"service.eventFirstSeen\"] = \"start_time\",\n [\"service.eventLastSeen\"] = \"end_time\",\n [\"createdAt\"] = \"metadata.original_time\",\n [\"updatedAt\"] = \"metadata.modified_time\",\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\nlocal function unknown_findings_ocsf_mapping()\n return {\n [\"class_uid\"] = \"class_uid\",\n [\"class_name\"] = \"class_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"category_name\"] = \"category_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"type_uid\"] = \"type_uid\",\n [\"type_name\"] = \"type_name\",\n [\"eventId\"] = \"event.type\",\n [\"severity_id\"] = \"severity_id\",\n [\"metadata.version\"] = \"metadata.version\",\n [\"metadata.product.name\"] = \"metadata.product.name\",\n [\"metadata.product.vendor_name\"] = \"metadata.product.vendor_name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"cloudProvider\"] = \"cloud.provider\",\n [\"accountTypeId\"] = \"cloud.account.type_id\",\n [\"cloudAccountType\"] = \"cloud.account.type\",\n [\"message\"] = \"message\",\n }\nend\n\n-- Finding types constants\nlocal ec2_finding_types = {\n [\"Backdoor:EC2/C&CActivity.B\"] = true,\n [\"Backdoor:EC2/C&CActivity.B!DNS\"] = true,\n [\"Backdoor:EC2/DenialOfService.Dns\"] = true,\n [\"Backdoor:EC2/DenialOfService.Tcp\"] = true,\n [\"Backdoor:EC2/DenialOfService.Udp\"] = true,\n [\"Backdoor:EC2/DenialOfService.UdpOnTcpPorts\"] = true,\n [\"Backdoor:EC2/DenialOfService.UnusualProtocol\"] = true,\n [\"Backdoor:EC2/Spambot\"] = true,\n [\"Behavior:EC2/NetworkPortUnusual\"] = true,\n [\"Behavior:EC2/TrafficVolumeUnusual\"] = true,\n [\"CryptoCurrency:EC2/BitcoinTool.B\"] = true,\n [\"CryptoCurrency:EC2/BitcoinTool.B!DNS\"] = true,\n [\"DefenseEvasion:EC2/UnusualDNSResolver\"] = true,\n [\"DefenseEvasion:EC2/UnusualDoHActivity\"] = true,\n [\"DefenseEvasion:EC2/UnusualDoTActivity\"] = true,\n [\"Impact:EC2/AbusedDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/BitcoinDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/MaliciousDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/PortSweep\"] = true,\n [\"Impact:EC2/SuspiciousDomainRequest.Reputation\"] = true,\n [\"Impact:EC2/WinRMBruteForce\"] = true,\n [\"Recon:EC2/PortProbeEMRUnprotectedPort\"] = true,\n [\"Recon:EC2/PortProbeUnprotectedPort\"] = true,\n [\"Recon:EC2/Portscan\"] = true,\n [\"Trojan:EC2/BlackholeTraffic\"] = true,\n [\"Trojan:EC2/BlackholeTraffic!DNS\"] = true,\n [\"Trojan:EC2/DGADomainRequest.B\"] = true,\n [\"Trojan:EC2/DGADomainRequest.C!DNS\"] = true,\n [\"Trojan:EC2/DNSDataExfiltration\"] = true,\n [\"Trojan:EC2/DriveBySourceTraffic!DNS\"] = true,\n [\"Trojan:EC2/DropPoint\"] = true,\n [\"Trojan:EC2/DropPoint!DNS\"] = true,\n [\"Trojan:EC2/PhishingDomainRequest!DNS\"] = true,\n [\"UnauthorizedAccess:EC2/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:EC2/MetadataDNSRebind\"] = true,\n [\"UnauthorizedAccess:EC2/RDPBruteForce\"] = true,\n [\"UnauthorizedAccess:EC2/SSHBruteForce\"] = true,\n [\"UnauthorizedAccess:EC2/TorClient\"] = true,\n [\"UnauthorizedAccess:EC2/TorRelay\"] = true\n}\n\nlocal runtime_monitoring_finding_types = {\n [\"CryptoCurrency:Runtime/BitcoinTool.B\"] = true,\n [\"Backdoor:Runtime/C&CActivity.B\"] = true,\n [\"UnauthorizedAccess:Runtime/TorRelay\"] = true,\n [\"UnauthorizedAccess:Runtime/TorClient\"] = true,\n [\"Trojan:Runtime/BlackholeTraffic\"] = true,\n [\"Trojan:Runtime/DropPoint\"] = true,\n [\"CryptoCurrency:Runtime/BitcoinTool.B!DNS\"] = true,\n [\"Backdoor:Runtime/C&CActivity.B!DNS\"] = true,\n [\"Trojan:Runtime/BlackholeTraffic!DNS\"] = true,\n [\"Trojan:Runtime/DropPoint!DNS\"] = true,\n [\"Trojan:Runtime/DGADomainRequest.C!DNS\"] = true,\n [\"Trojan:Runtime/DriveBySourceTraffic!DNS\"] = true,\n [\"Trojan:Runtime/PhishingDomainRequest!DNS\"] = true,\n [\"Impact:Runtime/AbusedDomainRequest.Reputation\"] = true,\n [\"Impact:Runtime/BitcoinDomainRequest.Reputation\"] = true,\n [\"Impact:Runtime/MaliciousDomainRequest.Reputation\"] = true,\n [\"Impact:Runtime/SuspiciousDomainRequest.Reputation\"] = true,\n [\"UnauthorizedAccess:Runtime/MetadataDNSRebind\"] = true,\n [\"Execution:Runtime/NewBinaryExecuted\"] = true,\n [\"PrivilegeEscalation:Runtime/DockerSocketAccessed\"] = true,\n [\"PrivilegeEscalation:Runtime/RuncContainerEscape\"] = true,\n [\"PrivilegeEscalation:Runtime/CGroupsReleaseAgentModified\"] = true,\n [\"DefenseEvasion:Runtime/ProcessInjection.Proc\"] = true,\n [\"DefenseEvasion:Runtime/ProcessInjection.Ptrace\"] = true,\n [\"DefenseEvasion:Runtime/ProcessInjection.VirtualMemoryWrite\"] = true,\n [\"Execution:Runtime/ReverseShell\"] = true,\n [\"DefenseEvasion:Runtime/FilelessExecution\"] = true,\n [\"Impact:Runtime/CryptoMinerExecuted\"] = true,\n [\"Execution:Runtime/NewLibraryLoaded\"] = true,\n [\"PrivilegeEscalation:Runtime/ContainerMountsHostDirectory\"] = true,\n [\"PrivilegeEscalation:Runtime/UserfaultfdUsage\"] = true\n}\n\nlocal iam_finding_types = {\n [\"CredentialAccess:IAMUser/AnomalousBehavior\"] = true,\n [\"DefenseEvasion:IAMUser/AnomalousBehavior\"] = true,\n [\"Discovery:IAMUser/AnomalousBehavior\"] = true,\n [\"Exfiltration:IAMUser/AnomalousBehavior\"] = true,\n [\"Impact:IAMUser/AnomalousBehavior\"] = true,\n [\"InitialAccess:IAMUser/AnomalousBehavior\"] = true,\n [\"PenTest:IAMUser/KaliLinux\"] = true,\n [\"PenTest:IAMUser/ParrotLinux\"] = true,\n [\"PenTest:IAMUser/PentooLinux\"] = true,\n [\"Persistence:IAMUser/AnomalousBehavior\"] = true,\n [\"Policy:IAMUser/RootCredentialUsage\"] = true,\n [\"PrivilegeEscalation:IAMUser/AnomalousBehavior\"] = true,\n [\"Recon:IAMUser/MaliciousIPCaller\"] = true,\n [\"Recon:IAMUser/MaliciousIPCaller.Custom\"] = true,\n [\"Recon:IAMUser/TorIPCaller\"] = true,\n [\"Stealth:IAMUser/CloudTrailLoggingDisabled\"] = true,\n [\"Stealth:IAMUser/PasswordPolicyChange\"] = true,\n [\"UnauthorizedAccess:IAMUser/ConsoleLoginSuccess.B\"] = true,\n [\"UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.InsideAWS\"] = true,\n [\"UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.OutsideAWS\"] = true,\n [\"UnauthorizedAccess:IAMUser/MaliciousIPCaller\"] = true,\n [\"UnauthorizedAccess:IAMUser/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:IAMUser/TorIPCaller\"] = true\n}\n\nlocal kubernetes_audit_logs_finding_types = {\n [\"CredentialAccess:Kubernetes/MaliciousIPCaller\"] = true,\n [\"CredentialAccess:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"CredentialAccess:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"CredentialAccess:Kubernetes/TorIPCaller\"] = true,\n [\"DefenseEvasion:Kubernetes/MaliciousIPCaller\"] = true,\n [\"DefenseEvasion:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"DefenseEvasion:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"DefenseEvasion:Kubernetes/TorIPCaller\"] = true,\n [\"Discovery:Kubernetes/MaliciousIPCaller\"] = true,\n [\"Discovery:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"Discovery:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"Discovery:Kubernetes/TorIPCaller\"] = true,\n [\"Execution:Kubernetes/ExecInKubeSystemPod\"] = true,\n [\"Impact:Kubernetes/MaliciousIPCaller\"] = true,\n [\"Impact:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"Impact:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"Impact:Kubernetes/TorIPCaller\"] = true,\n [\"Persistence:Kubernetes/ContainerWithSensitiveMount\"] = true,\n [\"Persistence:Kubernetes/MaliciousIPCaller\"] = true,\n [\"Persistence:Kubernetes/MaliciousIPCaller.Custom\"] = true,\n [\"Persistence:Kubernetes/SuccessfulAnonymousAccess\"] = true,\n [\"Persistence:Kubernetes/TorIPCaller\"] = true,\n [\"Policy:Kubernetes/AdminAccessToDefaultServiceAccount\"] = true,\n [\"Policy:Kubernetes/AnonymousAccessGranted\"] = true,\n [\"Policy:Kubernetes/ExposedDashboard\"] = true,\n [\"Policy:Kubernetes/KubeflowDashboardExposed\"] = true,\n [\"PrivilegeEscalation:Kubernetes/PrivilegedContainer\"] = true,\n [\"CredentialAccess:Kubernetes/AnomalousBehavior.SecretsAccessed\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleBindingCreated\"] = true,\n [\"Execution:Kubernetes/AnomalousBehavior.ExecInPod\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!PrivilegedContainer\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!ContainerWithSensitiveMount\"] = true,\n [\"Execution:Kubernetes/AnomalousBehavior.WorkloadDeployed\"] = true,\n [\"PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleCreated\"] = true,\n [\"Discovery:Kubernetes/AnomalousBehavior.PermissionChecked\"] = true\n}\n\nlocal lambda_protection_finding_types = {\n [\"Backdoor:Lambda/C&CActivity.B\"] = true,\n [\"CryptoCurrency:Lambda/BitcoinTool.B\"] = true,\n [\"Trojan:Lambda/BlackholeTraffic\"] = true,\n [\"Trojan:Lambda/DropPoint\"] = true,\n [\"UnauthorizedAccess:Lambda/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:Lambda/TorClient\"] = true,\n [\"UnauthorizedAccess:Lambda/TorRelay\"] = true\n}\n\nlocal malware_protection_finding_types = {\n [\"Execution:EC2/MaliciousFile\"] = true,\n [\"Execution:ECS/MaliciousFile\"] = true,\n [\"Execution:Kubernetes/MaliciousFile\"] = true,\n [\"Execution:Container/MaliciousFile\"] = true,\n [\"Execution:EC2/SuspiciousFile\"] = true,\n [\"Execution:ECS/SuspiciousFile\"] = true,\n [\"Execution:Kubernetes/SuspiciousFile\"] = true,\n [\"Execution:Container/SuspiciousFile\"] = true\n}\n\nlocal rds_protection_finding_types = {\n [\"CredentialAccess:RDS/AnomalousBehavior.SuccessfulLogin\"] = true,\n [\"CredentialAccess:RDS/AnomalousBehavior.FailedLogin\"] = true,\n [\"CredentialAccess:RDS/AnomalousBehavior.SuccessfulBruteForce\"] = true,\n [\"CredentialAccess:RDS/MaliciousIPCaller.SuccessfulLogin\"] = true,\n [\"CredentialAccess:RDS/MaliciousIPCaller.FailedLogin\"] = true,\n [\"Discovery:RDS/MaliciousIPCaller\"] = true,\n [\"CredentialAccess:RDS/TorIPCaller.SuccessfulLogin\"] = true,\n [\"CredentialAccess:RDS/TorIPCaller.FailedLogin\"] = true,\n [\"Discovery:RDS/TorIPCaller\"] = true\n}\n\nlocal s3_finding_types = {\n [\"Discovery:S3/AnomalousBehavior\"] = true,\n [\"Discovery:S3/MaliciousIPCaller\"] = true,\n [\"Discovery:S3/MaliciousIPCaller.Custom\"] = true,\n [\"Discovery:S3/TorIPCaller\"] = true,\n [\"Exfiltration:S3/AnomalousBehavior\"] = true,\n [\"Exfiltration:S3/MaliciousIPCaller\"] = true,\n [\"Impact:S3/AnomalousBehavior.Delete\"] = true,\n [\"Impact:S3/AnomalousBehavior.Permission\"] = true,\n [\"Impact:S3/AnomalousBehavior.Write\"] = true,\n [\"Impact:S3/MaliciousIPCaller\"] = true,\n [\"PenTest:S3/KaliLinux\"] = true,\n [\"PenTest:S3/ParrotLinux\"] = true,\n [\"PenTest:S3/PentooLinux\"] = true,\n [\"Policy:S3/AccountBlockPublicAccessDisabled\"] = true,\n [\"Policy:S3/BucketAnonymousAccessGranted\"] = true,\n [\"Policy:S3/BucketBlockPublicAccessDisabled\"] = true,\n [\"Policy:S3/BucketPublicAccessGranted\"] = true,\n [\"Stealth:S3/ServerAccessLoggingDisabled\"] = true,\n [\"UnauthorizedAccess:S3/MaliciousIPCaller.Custom\"] = true,\n [\"UnauthorizedAccess:S3/TorIPCaller\"] = true\n}\n\n\n-- Per-category helpers (mirroring Python OCSFMapperHelper)\nlocal function ec2_findings(event, site_id)\n event[\"class_uid\"] = 4001\n event[\"class_name\"] = \"Network Activity\"\n event[\"category_uid\"] = 4\n event[\"category_name\"] = \"Network Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"EC2 finding types\"\n event[\"type_uid\"] = 400199\n event[\"type_name\"] = \"EC2 finding types\"\n event[\"eventId\"] = \"EC2 finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, ec2_findings_ocsf_mapping())\n -- enrich with fields that were set directly on event\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function runtime_monitoring_findings(event, site_id)\n event[\"class_uid\"] = 6003\n event[\"class_name\"] = \"API Activity\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"EKS Runtime Monitoring finding types\"\n event[\"type_uid\"] = 600399\n event[\"type_name\"] = \"EKS Runtime Monitoring finding types\"\n event[\"eventId\"] = \"EKS Runtime Monitoring finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, runtime_monitoring_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function iam_findings(event, site_id)\n event[\"class_uid\"] = 3004\n event[\"class_name\"] = \"Entity Management\"\n event[\"category_uid\"] = 3\n event[\"category_name\"] = \"Identity & Access Management\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"IAM finding types\"\n event[\"type_uid\"] = 300499\n event[\"type_name\"] = \"IAM finding types\"\n event[\"eventId\"] = \"IAM finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, iam_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function kubernetes_audit_log_findings(event, site_id)\n event[\"class_uid\"] = 6003\n event[\"class_name\"] = \"API Activity\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"Kubernetes Audit Logs finding types\"\n event[\"type_uid\"] = 600399\n event[\"type_name\"] = \"Kubernetes Audit Logs finding types\"\n event[\"eventId\"] = \"Kubernetes Audit Logs finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, kubernetes_audit_log_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function lambda_findings(event, site_id)\n event[\"class_uid\"] = 6002\n event[\"class_name\"] = \"Application Lifecycle\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"Lambda Protection finding types\"\n event[\"type_uid\"] = 600299\n event[\"type_name\"] = \"Lambda Protection finding types\"\n event[\"eventId\"] = \"Lambda Protection finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, lambda_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function malware_findings(event, site_id)\n event[\"class_uid\"] = 2001\n event[\"class_name\"] = \"Security Finding\"\n event[\"category_uid\"] = 2\n event[\"category_name\"] = \"Findings\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"Malware Protection finding types\"\n event[\"type_uid\"] = 200199\n event[\"type_name\"] = \"Malware Protection finding types\"\n event[\"eventId\"] = \"Malware Protection finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, malware_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n return flat\nend\n\nlocal function rds_findings(event, site_id)\n event[\"class_uid\"] = 3002\n event[\"class_name\"] = \"Authentication\"\n event[\"category_uid\"] = 3\n event[\"category_name\"] = \"Identity & Access Management\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"RDS Protection finding types\"\n event[\"type_uid\"] = 300299\n event[\"type_name\"] = \"RDS Protection finding types\"\n event[\"eventId\"] = \"RDS Protection finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, rds_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function s3_findings(event, site_id)\n event[\"class_uid\"] = 6003\n event[\"class_name\"] = \"API Activity\"\n event[\"category_uid\"] = 6\n event[\"category_name\"] = \"Application Activity\"\n event[\"activity_id\"] = \"99\"\n event[\"activity_name\"] = \"S3 finding types\"\n event[\"type_uid\"] = 600399\n event[\"type_name\"] = \"S3 finding types\"\n event[\"eventId\"] = \"S3 finding types\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, s3_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n coordinates_mapping(flat)\n return flat\nend\n\nlocal function unknown_findings(event, site_id)\n event[\"class_uid\"] = 0\n event[\"class_name\"] = \"Base Event\"\n event[\"category_uid\"] = 0\n event[\"category_name\"] = \"Uncategorized\"\n event[\"activity_id\"] = 0\n event[\"activity_name\"] = \"Unknown\"\n event[\"type_uid\"] = 0\n event[\"type_name\"] = \"Unknown\"\n event[\"eventId\"] = \"Unknown\"\n common_mapping(event, site_id)\n local flat = apply_mapping(event, unknown_findings_ocsf_mapping())\n flat[\"activity_name\"] = event[\"activity_name\"]\n flat[\"activity_id\"] = event[\"activity_id\"]\n flat[\"class_uid\"] = event[\"class_uid\"]\n flat[\"class_name\"] = event[\"class_name\"]\n flat[\"category_uid\"] = event[\"category_uid\"]\n flat[\"category_name\"] = event[\"category_name\"]\n flat[\"type_uid\"] = event[\"type_uid\"]\n flat[\"type_name\"] = event[\"type_name\"]\n flat[\"severity_id\"] = event[\"severity_id\"]\n return flat\nend\n\n-- Dispatcher. family one of: \"ec2\",\"runtime\",\"iam\",\"kubernetes\",\"lambda\",\"malware\",\"rds\",\"s3\",\"unknown\"\nlocal function processGuardDutyEvent(event, site_id, family)\n local fam = tostring(family or \"unknown\"):lower()\n if fam == \"ec2\" then return ec2_findings(deepCopy(event), site_id) end\n if fam == \"runtime\" or fam == \"eks_runtime\" or fam == \"runtime_monitoring\" then return runtime_monitoring_findings(deepCopy(event), site_id) end\n if fam == \"iam\" then return iam_findings(deepCopy(event), site_id) end\n if fam == \"kubernetes\" or fam == \"k8s\" then return kubernetes_audit_log_findings(deepCopy(event), site_id) end\n if fam == \"lambda\" then return lambda_findings(deepCopy(event), site_id) end\n if fam == \"malware\" then return malware_findings(deepCopy(event), site_id) end\n if fam == \"rds\" then return rds_findings(deepCopy(event), site_id) end\n if fam == \"s3\" then return s3_findings(deepCopy(event), site_id) end\n return unknown_findings(deepCopy(event), site_id)\nend\n\n-- Best-effort automatic family detection from GuardDuty event structure\nlocal function detect_family(event)\n local type = getByPath(event, {\"type\"})\n if ec2_finding_types[type] then return \"ec2\" end\n if runtime_monitoring_finding_types[type] then return \"runtime\" end\n if iam_finding_types[type] then return \"iam\" end\n if kubernetes_audit_logs_finding_types[type] then return \"kubernetes\" end\n if lambda_protection_finding_types[type] then return \"lambda\" end\n if malware_protection_finding_types[type] then return \"malware\" end\n if rds_protection_finding_types[type] then return \"rds\" end\n if s3_finding_types[type] then return \"s3\" end\n return \"unknown\"\nend\n\n-- Helper: Encode Lua table to JSON string with field ordering\nlocal function encodeJson(obj, fieldOrder, key)\n if obj == nil then\n\t return \"null\"\n elseif type(obj) == \"boolean\" then\n\t return tostring(obj)\n elseif type(obj) == \"number\" then\n\t return tostring(obj)\n elseif type(obj) == \"string\" then\n\t return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n\t local isArray = true\n\t local maxIndex = 0\n\t for k, v in pairs(obj) do\n\t\t if type(k) ~= \"number\" then\n\t\t\t isArray = false\n\t\t\t break\n\t\t end\n\t\t maxIndex = math.max(maxIndex, k)\n\t end\n\t if isArray then\n\t\t local items = {}\n\t\t for i = 1, maxIndex do\n\t\t\t table.insert(items, encodeJson(obj[i], fieldOrder, key))\n\t\t end\n\t\t return \"[\" .. table.concat(items, \",\") .. \"]\"\n\t else\n\t\t local items = {}\n\t\t local fieldOrdering = fieldOrder[key] or {}\n\t\t \n\t\t -- Phase 1: ordered keys\n\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t local v = obj[fieldName]\n\t\t\t if v ~= nil then\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, fieldName)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t -- Phase 2: remaining keys (not in fieldOrder)\n\t\t for k, v in pairs(obj) do\n\t\t\t local found = false\n\t\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t\t if k == fieldName then\n\t\t\t\t\t found = true\n\t\t\t\t\t break\n\t\t\t\t end\n\t\t\t end\n\t\t\t if not found then\n\t\t\t\t local keyStr = type(k) == \"string\" and k or tostring(k)\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, keyStr)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t return \"{\" .. table.concat(items, \",\") .. \"}\"\n\t end\n else\n\t return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal function convertUtcToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\nlocal IGNORE_KEYS = {\n _ob = true,\n timestamp = true, -- Ignore timestamp as we use start_time/end_time\n}\n\n-- Global entry point expected by transform runners\nfunction processEvent(event)\n local site_id = nil\n local site_tbl = event and event[\"site\"]\n if type(site_tbl) == \"table\" and site_tbl[\"id\"] then\n site_id = site_tbl[\"id\"]\n elseif event and event[\"site_id\"] then\n site_id = event[\"site_id\"]\n end\n local fam = detect_family(event or {})\n local copy = deepCopy(event, IGNORE_KEYS)\n local result = processGuardDutyEvent(copy, site_id, fam)\n result.time = convertUtcToMilliseconds(copy[\"createdAt\"])\n result.message = encodeJson(copy, get_msg_field_ordering(fam), \"message\")\n return result\nend\n\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "F", - "harness_score": 45, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_guardduty/metadata.yaml b/pipelines/community/transform_ocsf/aws_guardduty/metadata.yaml deleted file mode 100644 index 92883b6..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/metadata.yaml +++ /dev/null @@ -1,51 +0,0 @@ -grade: - letter: F - score: 45 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Aws Guardduty. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: aws - dataSource: Aws Guardduty - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "Other - {Explain: AWS EventBridge or S3 export of GuardDuty findings}" - auth_type: "IAM Role" - sample_record: "{\n \"schemaVersion\": \"2.0\",\n \"accountId\": \"222708836859\",\n \"region\":\ - \ \"us-east-1\",\n \"partition\": \"aws\",\n \"id\": \"8db850b8-f1b1-4dfd-8676-efd7b4e8ee85\",\n\ - \ \"arn\": \"arn:aws:guardduty:us-east-1::84378c5c8013403891eb51ada1b2a47b:detector/84378c5c8013403891eb51ada1b2a47b/finding/8db850b8-f1b1-4dfd-8676-efd7b4e8ee85\"\ - ,\n \"type\": \"UnauthorizedAccess:IAMUser/ConsoleLogin\",\n \"title\": \"EC2 instance attempting\ - \ connection to a blackholed IP address.\",\n \"description\": \"EC2 instance is attempting to communicate\ - \ with a blackholed IP on port 80.\",\n \"severity\": 8.0,\n \"createdAt\": \"2026-04-20T03:40:52Z\"\ - ,\n \"updatedAt\": \"2026-04-20T03:40:52Z\",\n \"resource\": {\n \"resourceType\": \"Instance\"\ - ,\n \"instanceDetails\": {\n \"instanceId\": \"i-5c365f7b\",\n \"instanceType\": \"m5.large\"\ - ,\n \"platform\": null,\n \"networkInterfaces\": [\n {\n \"networkInterfaceId\"\ - : \"eni-26b50034\",\n \"privateIpAddress\": \"219.13.151.53\",\n \"publicIp\": \"\ - 19.41.200.229\",\n \"ipv6Addresses\": []\n }\n ],\n \"tags\": []\n }\n\ - \ },\n \"service\": {\n \"serviceName\": \"guardduty\",\n \"detectorId\": \"84378c5c8013403891eb51ada1b2a47b\"\ - ,\n \"eventFirstSeen\": \"2026-04-20T03:40:52Z\",\n \"eventLastSeen\": \"2026-04-20T03:40:52Z\"\ - ,\n \"count\": 1,\n \"action\": {\n \"actionType\": \"NETWORK_CONNECTION\",\n \"networkConnectionAction\"\ - : {\n \"connectionDirection\": \"OUTBOUND\",\n \"protocol\": \"TCP\",\n \"port\"\ - : 80,\n \"remoteIpDetails\": {\n \"ipAddressV4\": \"19" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: aws, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: F - harness_score: 45 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/aws_guardduty/sample.json b/pipelines/community/transform_ocsf/aws_guardduty/sample.json deleted file mode 100644 index 6c70780..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/sample.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "schemaVersion": "2.0", - "accountId": "222708836859", - "region": "us-east-1", - "partition": "aws", - "id": "8db850b8-f1b1-4dfd-8676-efd7b4e8ee85", - "arn": "arn:aws:guardduty:us-east-1::84378c5c8013403891eb51ada1b2a47b:detector/84378c5c8013403891eb51ada1b2a47b/finding/8db850b8-f1b1-4dfd-8676-efd7b4e8ee85", - "type": "UnauthorizedAccess:IAMUser/ConsoleLogin", - "title": "EC2 instance attempting connection to a blackholed IP address.", - "description": "EC2 instance is attempting to communicate with a blackholed IP on port 80.", - "severity": 8.0, - "createdAt": "2026-04-20T03:40:52Z", - "updatedAt": "2026-04-20T03:40:52Z", - "resource": { - "resourceType": "Instance", - "instanceDetails": { - "instanceId": "i-5c365f7b", - "instanceType": "m5.large", - "platform": null, - "networkInterfaces": [ - { - "networkInterfaceId": "eni-26b50034", - "privateIpAddress": "219.13.151.53", - "publicIp": "19.41.200.229", - "ipv6Addresses": [] - } - ], - "tags": [] - } - }, - "service": { - "serviceName": "guardduty", - "detectorId": "84378c5c8013403891eb51ada1b2a47b", - "eventFirstSeen": "2026-04-20T03:40:52Z", - "eventLastSeen": "2026-04-20T03:40:52Z", - "count": 1, - "action": { - "actionType": "NETWORK_CONNECTION", - "networkConnectionAction": { - "connectionDirection": "OUTBOUND", - "protocol": "TCP", - "port": 80, - "remoteIpDetails": { - "ipAddressV4": "190.60.202.107", - "organization": { - "asn": "-1", - "asnOrg": "GeneratedASNOrg", - "isp": "GeneratedISP", - "org": "GeneratedORG" - } - } - } - } - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/aws_guardduty/serializer.lua b/pipelines/community/transform_ocsf/aws_guardduty/serializer.lua deleted file mode 100644 index f715edd..0000000 --- a/pipelines/community/transform_ocsf/aws_guardduty/serializer.lua +++ /dev/null @@ -1,1720 +0,0 @@ --- AWS GuardDuty OCSF 1.0.0 parser (ported from Python) - --- Helper: split string by delimiter -local function split(str, delimiter) - local result = {} - local escaped = delimiter:gsub("[%.%+%*%?%^%$%(%)%[%]%%]", "%%%1") - local pattern = "([^" .. escaped .. "]+)" - for token in tostring(str):gmatch(pattern) do - table.insert(result, token) - end - if #result == 0 and #tostring(str) > 0 then - table.insert(result, str) - end - return result -end - --- Helper: safely access nested keys: keys is array, obj is table -local function getByPath(obj, keys) - local current = obj - for _, k in ipairs(keys) do - if current ~= nil and type(current) == "table" then - current = current[k] - else - return nil - end - end - return current -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - --- Common mapping applied before specific finders -local function common_mapping(event, site_id) - event["dataSource"] = { category = "security", name = "AWS GuardDuty", vendor = "AWS" } - event["metadata"] = { version = "1.0.0", product = { name = "AWS GuardDuty", vendor_name = "AWS" } } - event["cloudProvider"] = "AWS" - event["cloudAccountType"] = "AWS Account" - event["accountTypeId"] = 10 - if site_id ~= nil and site_id ~= "" then - event["site"] = { id = site_id } - end - local sev = event["severity"] - if type(sev) == "number" then - if sev >= 1.0 and sev <= 2.9 then - event["severity_id"] = 2 - elseif sev >= 4.0 and sev <= 6.9 then - event["severity_id"] = 3 - elseif sev >= 7.0 and sev <= 8.9 then - event["severity_id"] = 4 - else - event["severity_id"] = 0 - end - end -end - --- After mapping, convert lat/lon into coordinates and remove lat/lon fields based on activity_name -local function coordinates_mapping(parsed) - local lat = parsed["lat"] - local lon = parsed["lon"] - if (type(lat) == "number" or type(lat) == "string") or (type(lon) == "number" or type(lon) == "string") then - local latnum = tonumber(lat) - local lonnum = tonumber(lon) - if latnum and lonnum then - local coords = { latnum, lonnum } - local activity = parsed["activity_name"] - if activity == "EC2 finding types" or - activity == "EKS Runtime Monitoring finding types" or - activity == "Kubernetes Audit Logs finding types" or - activity == "RDS Protection finding types" or - activity == "S3 finding types" then - parsed["src_endpoint.location.coordinates"] = coords - parsed["lat"], parsed["lon"] = nil, nil - elseif activity == "IAM finding types" or activity == "Lambda Protection finding types" then - parsed["device.location.coordinates"] = coords - parsed["lat"], parsed["lon"] = nil, nil - end - end - end -end - -local function flatten_table(tbl, prefix, out) - out = out or {} - prefix = prefix or "" - local hasArrayElements = false - - for k, v in pairs(tbl) do - if type(k) == "number" then - hasArrayElements = true - break - end - end - - if hasArrayElements then - -- This is an array, store it as-is - if prefix ~= "" then - out[prefix] = tbl - end - return out - end - - for k, v in pairs(tbl) do - local key = prefix ~= "" and (prefix .. "." .. k) or k - if type(v) == "table" then - flatten_table(v, key, out) - else - out[key] = v - end - end - - return out -end - --- Apply a mapping table of { [source_dotted] = target_dotted } -local function apply_mapping(event, mapping) - local out = {} - for src, dst in pairs(mapping) do - local val = getByPath(event, split(src, ".")) - if val ~= nil then - out[dst] = val - end - end - - -- NEW: auto-capture unmapped fields - local copy = deepCopy(event) - local flattenCopy = flatten_table(copy) - for key, val in pairs(flattenCopy) do - if mapping[key] == nil and val ~= nil and val ~= "" and val ~= "[]" and val ~= "()" and val ~= "{}" then - out["unmapped." .. key] = val - end - end - return out -end - -local EC2_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition","id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "instanceDetails" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", - "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "networkConnectionAction" - }, - networkConnectionAction = { - "connectionDirection", "localIpDetails", "remoteIpDetails", "remotePortDetails", - "localPortDetails", "protocol", "blocked" - }, - localIpDetails = { - "ipAddressV4" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - remotePortDetails = { - "port", "portName" - }, - localPortDetails = { - "port", "portName" - }, - additionalInfo = { - "threatListName", "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local EKS_RUNTIME_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "eksClusterDetails", "kubernetesDetails", "containerDetails", "instanceDetails" - }, - eksClusterDetails = { - "name", "arn", "createdAt", "vpcId", "status", "tags" - }, - tags = { - "key", "value" - }, - kubernetesDetails = { - "kubernetesWorkloadDetails" - }, - kubernetesWorkloadDetails = { - "name", "namespace", "type", "uid" - }, - containerDetails = { - "id", "name", "image" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - service = { - "serviceName", "detectorId", "action", "runtimeDetails", "featureName", "resourceRole", - "additionalInfo", "evidence", "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "dnsRequestAction" - }, - dnsRequestAction = { - "domain", "protocol", "blocked", "domainWithSuffix" - }, - runtimeDetails = { - "process" - }, - process = { - "pid", "name", "uuid", "executablePath", "executableSha256", "cmdLine", "user", "euid", - "userId", "pwd", "startTime", "parentUuid", "lineage" - }, - lineage = { - "pid", "uuid", "executablePath", "euid", "parentUuid" - }, - additionalInfo = { - "threatListName", "sample", "agentDetails", "value", "type" - }, - agentDetails = { - "agentVersion", "agentId" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local IAM_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "accessKeyDetails", "instanceDetails" - }, - accessKeyDetails = { - "accessKeyId", "principalId", "userType", "userName" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "awsApiCallAction" - }, - awsApiCallAction = { - "api", "serviceName", "callerType", "errorCode", "remoteIpDetails", "affectedResources" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "userAgent", "anomalies", "profiledBehavior", "unusualBehavior", "sample", "value", "type" - }, - userAgent = { - "fullUserAgent", "userAgentCategory" - }, - anomalies = { - "anomalousAPIs" - }, - profiledBehavior = { - "rareProfiledAPIsAccountProfiling", "infrequentProfiledAPIsAccountProfiling", - "frequentProfiledAPIsAccountProfiling", "rareProfiledAPIsUserIdentityProfiling", - "infrequentProfiledAPIsUserIdentityProfiling", "frequentProfiledAPIsUserIdentityProfiling", - "rareProfiledUserTypesAccountProfiling", "infrequentProfiledUserTypesAccountProfiling", - "frequentProfiledUserTypesAccountProfiling", "rareProfiledUserNamesAccountProfiling", - "infrequentProfiledUserNamesAccountProfiling", "frequentProfiledUserNamesAccountProfiling", - "rareProfiledASNsAccountProfiling", "infrequentProfiledASNsAccountProfiling", - "frequentProfiledASNsAccountProfiling", "rareProfiledASNsUserIdentityProfiling", - "infrequentProfiledASNsUserIdentityProfiling", "frequentProfiledASNsUserIdentityProfiling", - "rareProfiledUserAgentsAccountProfiling", "infrequentProfiledUserAgentsAccountProfiling", - "frequentProfiledUserAgentsAccountProfiling", "rareProfiledUserAgentsUserIdentityProfiling", - "infrequentProfiledUserAgentsUserIdentityProfiling", "frequentProfiledUserAgentsUserIdentityProfiling" - }, - unusualBehavior = { - "unusualAPIsAccountProfiling", "unusualAPIsUserIdentityProfiling", - "unusualUserTypesAccountProfiling", "unusualUserNamesAccountProfiling", - "unusualASNsAccountProfiling", "unusualASNsUserIdentityProfiling", - "unusualUserAgentsAccountProfiling", "unusualUserAgentsUserIdentityProfiling", - "isUnusualUserIdentity" - }, -} - -local KUBERNETES_AUDIT_LOGS_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "eksClusterDetails", "kubernetesDetails", "accessKeyDetails" - }, - eksClusterDetails = { - "name", "arn", "createdAt", "vpcId", "status", "tags" - }, - tags = { - "key", "value" - }, - kubernetesDetails = { - "kubernetesWorkloadDetails", "kubernetesUserDetails" - }, - kubernetesWorkloadDetails = { - "name", "namespace", "type", "uid" - }, - kubernetesUserDetails = { - "username", "uid", "groups", "sessionName" - }, - accessKeyDetails = { - "accessKeyId", "principalId", "userType", "userName" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "kubernetesApiCallAction" - }, - kubernetesApiCallAction = { - "requestUri", "verb", "sourceIPs", "userAgent", "remoteIpDetails", "statusCode" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "threatName", "threatListName", "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local LAMBDA_PROTECTION_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "lambdaDetails" - }, - lambdaDetails = { - "functionArn", "functionName", "description", "lastModifiedAt", "revisionId", - "functionVersion", "role", "vpcConfig", "tags" - }, - vpcConfig = { - "vpcId", "securityGroups", "subnetIds" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "networkConnectionAction" - }, - networkConnectionAction = { - "connectionDirection", "remoteIpDetails", "remotePortDetails", "protocol", "blocked" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - remotePortDetails = { - "port", "portName" - }, - additionalInfo = { - "unusualProtocol", "threatListName", "unusual", "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local MALWARE_PROTECTION_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "instanceDetails", "ebsVolumeDetails" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - tags = { - "key", "value" - }, - ebsVolumeDetails = { - "scannedVolumeDetails", "skippedVolumeDetails" - }, - scannedVolumeDetails = { - "volumeArn", "volumeType", "deviceName", "volumeSizeInGB", "encryptionType", "snapshotArn", "kmsKeyArn" - }, - service = { - "serviceName", "detectorId", "featureName", "ebsVolumeScanDetails", "additionalInfo", "evidence", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - ebsVolumeScanDetails = { - "scanId", "scanStartedAt", "scanCompletedAt", "scanType", "triggerFindingId", "sources", "scanDetections" - }, - scanDetections = { - "scannedItemCount", "threatsDetectedItemCount", "highestSeverityThreatDetails", "threatDetectedByName" - }, - scannedItemCount = { - "totalGb", "files", "volumes" - }, - threatsDetectedItemCount = { - "files" - }, - highestSeverityThreatDetails = { - "severity", "threatName", "count" - }, - threatDetectedByName = { - "itemCount", "uniqueThreatNameCount", "shortened", "threatNames" - }, - threatNames = { - "name", "severity", "itemCount", "filePaths" - }, - filePaths = { - "filePath", "fileName", "volumeArn", "hash" - }, - additionalInfo = { - "sample", "value", "type" - }, -} - -local RDS_PROTECTION_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "rdsDbInstanceDetails", "rdsDbUserDetails" - }, - rdsDbInstanceDetails = { - "dbInstanceIdentifier", "engine", "engineVersion", "dbClusterIdentifier", "dbInstanceArn", "tags" - }, - tags = { - "key", "value" - }, - rdsDbUserDetails = { - "user", "application", "database", "ssl", "authMethod" - }, - service = { - "action", "additionalInfo", "resourceRole", "evidence", "count", "detectorId", - "eventFirstSeen", "eventLastSeen", "serviceName", "archived" - }, - action = { - "actionType", "rdsLoginAttemptAction" - }, - rdsLoginAttemptAction = { - "remoteIpDetails" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "sample", "value", "type" - }, - evidence = { - "threatIntelligenceDetails" - }, - threatIntelligenceDetails = { - "threatListName", "threatNames" - }, -} - -local S3_FIELD_ORDER = { - message = { - "schemaVersion", "accountId", "region", "partition", "id", "arn", "type", "resource", - "service", "severity", "createdAt", "updatedAt", "title", "description" - }, - resource = { - "resourceType", "accessKeyDetails", "s3BucketDetails", "instanceDetails" - }, - accessKeyDetails = { - "accessKeyId", "principalId", "userType", "userName" - }, - s3BucketDetails = { - "arn", "name", "type", "createdAt", "owner", "tags", "defaultServerSideEncryption", "publicAccess" - }, - owner = { - "id" - }, - tags = { - "key", "value" - }, - defaultServerSideEncryption = { - "encryptionType", "kmsMasterKeyArn" - }, - publicAccess = { - "permissionConfiguration", "effectivePermission" - }, - permissionConfiguration = { - "bucketLevelPermissions", "accountLevelPermissions" - }, - bucketLevelPermissions = { - "accessControlList", "bucketPolicy", "blockPublicAccess" - }, - accessControlList = { - "allowsPublicReadAccess", "allowsPublicWriteAccess" - }, - bucketPolicy = { - "allowsPublicReadAccess", "allowsPublicWriteAccess" - }, - blockPublicAccess = { - "ignorePublicAcls", "restrictPublicBuckets", "blockPublicAcls", "blockPublicPolicy" - }, - accountLevelPermissions = { - "blockPublicAccess" - }, - instanceDetails = { - "instanceId", "instanceType", "outpostArn", "launchTime", "platform", "productCodes", - "iamInstanceProfile", "networkInterfaces", "tags", "instanceState", "availabilityZone", - "imageId", "imageDescription" - }, - productCodes = { - "productCodeId", "productCodeType" - }, - iamInstanceProfile = { - "arn", "id" - }, - networkInterfaces = { - "ipv6Addresses", "networkInterfaceId", "privateDnsName", "privateIpAddress", - "privateIpAddresses", "subnetId", "vpcId", "securityGroups", "publicDnsName", "publicIp" - }, - privateIpAddresses = { - "privateDnsName", "privateIpAddress" - }, - securityGroups = { - "groupName", "groupId" - }, - service = { - "serviceName", "detectorId", "action", "resourceRole", "additionalInfo", - "eventFirstSeen", "eventLastSeen", "archived", "count" - }, - action = { - "actionType", "awsApiCallAction" - }, - awsApiCallAction = { - "api", "serviceName", "callerType", "errorCode", "remoteIpDetails", "affectedResources" - }, - remoteIpDetails = { - "ipAddressV4", "organization", "country", "city", "geoLocation" - }, - organization = { - "asn", "asnOrg", "isp", "org" - }, - country = { - "countryName" - }, - city = { - "cityName" - }, - geoLocation = { - "lat", "lon" - }, - additionalInfo = { - "unusual", "sample", "value", "type" - }, - unusual = { - "hoursOfDay", "userNames" - }, -} - -local function get_msg_field_ordering(family) - if family == "ec2" then return EC2_FIELD_ORDER end - if family == "runtime" or family == "eks_runtime" or family == "runtime_monitoring" then return EKS_RUNTIME_FIELD_ORDER end - if family == "iam" then return IAM_FIELD_ORDER end - if family == "kubernetes" or family == "k8s" then return KUBERNETES_AUDIT_LOGS_FIELD_ORDER end - if family == "lambda" then return LAMBDA_PROTECTION_FIELD_ORDER end - if family == "malware" then return MALWARE_PROTECTION_FIELD_ORDER end - if family == "rds" then return RDS_PROTECTION_FIELD_ORDER end - if family == "s3" then return S3_FIELD_ORDER end - return {} -end - --- OCSF Mappings (ported 1:1 from Python) -local function ec2_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.instanceDetails.instanceId"] = "device.instance_uid", - ["resource.instanceDetails.networkInterfaces"] = "device.network_interfaces", - ["resource.instanceDetails.imageId"] = "device.image.uid", - ["service.serviceName"] = "src_endpoint.svc_name", - ["service.detectorId"] = "src_endpoint.uid", - ["service.action.networkConnectionAction.localIpDetails.ipAddressV4"] = "src_endpoint.ip", - ["service.action.networkConnectionAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.networkConnectionAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.networkConnectionAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.action.networkConnectionAction.localPortDetails.port"] = "src_endpoint.port", - ["service.action.networkConnectionAction.protocol"] = "connection_info.protocol_name", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function runtime_monitoring_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.instanceDetails.instanceId"] = "device.instance_uid", - ["resource.instanceDetails.networkInterfaces"] = "device.network_interfaces", - ["resource.instanceDetails.imageId"] = "device.image.uid", - ["service.runtimeDetails.process.pid"] = "actor.process.uid", - ["service.runtimeDetails.process.name"] = "actor.process.name", - ["service.runtimeDetails.process.user"] = "actor.user.name", - ["service.runtimeDetails.process.userId"] = "actor.user.uid", - ["service.runtimeDetails.process.parentUuid"] = "actor.process.parent_process.uid", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.featureName"] = "api.service.name", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function iam_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.accessKeyDetails.accessKeyId"] = "actor.session.credential_uid", - ["resource.accessKeyDetails.principalId"] = "actor.session.uid", - ["resource.accessKeyDetails.userName"] = "actor.user.name", - ["resource.instanceDetails.instanceId"] = "device.instance_uid", - ["resource.instanceDetails.networkInterfaces"] = "device.network_interfaces", - ["resource.instanceDetails.imageId"] = "device.image.uid", - ["service.action.awsApiCallAction.serviceName"] = "api.service.name", - ["service.action.awsApiCallAction.remoteIpDetails.organization.isp"] = "device.location.isp", - ["service.action.awsApiCallAction.remoteIpDetails.country.countryName"] = "device.location.country", - ["service.action.awsApiCallAction.remoteIpDetails.city.cityName"] = "device.location.city", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function kubernetes_audit_log_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.kubernetesDetails.kubernetesUserDetails.username"] = "actor.user.name", - ["resource.kubernetesDetails.kubernetesUserDetails.uid"] = "actor.user.uid", - ["resource.kubernetesDetails.kubernetesUserDetails.groups"] = "actor.user.groups", - ["resource.accessKeyDetails.accessKeyId"] = "actor.session.credential_uid", - ["resource.accessKeyDetails.principalId"] = "actor.session.uid", - ["service.serviceName"] = "src_endpoint.svc_name", - ["service.detectorId"] = "src_endpoint.uid", - ["service.action.kubernetesApiCallAction.requestUri"] = "http_request.url.url_string", - ["service.action.kubernetesApiCallAction.verb"] = "http_request.http_method", - ["service.action.kubernetesApiCallAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.kubernetesApiCallAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.kubernetesApiCallAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.kubernetesApiCallAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.action.kubernetesApiCallAction.statusCode"] = "status_code", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function lambda_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["service.action.networkConnectionAction.remoteIpDetails.organization.isp"] = "device.location.isp", - ["service.action.networkConnectionAction.remoteIpDetails.country.countryName"] = "device.location.country", - ["service.action.networkConnectionAction.remoteIpDetails.city.cityName"] = "device.location.city", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.networkConnectionAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function malware_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["partition"] = "resources.cloud_partition", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["service.featureName"] = "api.service.name", - ["service.ebsVolumeScanDetails.scanId"] = "api.service.uid", - ["service.ebsVolumeScanDetails.scanStartedAt"] = "finding.first_seen_time", - ["service.ebsVolumeScanDetails.scanCompletedAt"] = "finding.last_seen_time", - ["service.ebsVolumeScanDetails.triggerFindingId"] = "finding.uid", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function rds_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.rdsLoginAttemptAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.detectorId"] = "src_endpoint.uid", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["service.serviceName"] = "src_endpoint.svc_name", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function s3_findings_ocsf_mapping() - return { - ["schemaVersion"] = "metadata.log_version", - ["accountId"] = "cloud.account.uid", - ["region"] = "cloud.region", - ["partition"] = "resources.cloud_partition", - ["id"] = "metadata.uid", - ["type"] = "metadata.log_name", - ["resource.resourceType"] = "metadata.log_provider", - ["resource.accessKeyDetails.accessKeyId"] = "resources.user.credential_uid", - ["resource.accessKeyDetails.userType"] = "resources.user.type", - ["resource.accessKeyDetails.userName"] = "resources.user.name", - ["service.serviceName"] = "src_endpoint.svc_name", - ["service.detectorId"] = "src_endpoint.uid", - ["service.action.awsApiCallAction.serviceName"] = "api.service.name", - ["service.action.awsApiCallAction.remoteIpDetails.organization.isp"] = "src_endpoint.location.isp", - ["service.action.awsApiCallAction.remoteIpDetails.country.countryName"] = "src_endpoint.location.country", - ["service.action.awsApiCallAction.remoteIpDetails.city.cityName"] = "src_endpoint.location.city", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lat"] = "lat", - ["service.action.awsApiCallAction.remoteIpDetails.geoLocation.lon"] = "lon", - ["service.eventFirstSeen"] = "start_time", - ["service.eventLastSeen"] = "end_time", - ["createdAt"] = "metadata.original_time", - ["updatedAt"] = "metadata.modified_time", - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - -local function unknown_findings_ocsf_mapping() - return { - ["class_uid"] = "class_uid", - ["class_name"] = "class_name", - ["category_uid"] = "category_uid", - ["category_name"] = "category_name", - ["activity_id"] = "activity_id", - ["activity_name"] = "activity_name", - ["type_uid"] = "type_uid", - ["type_name"] = "type_name", - ["eventId"] = "event.type", - ["severity_id"] = "severity_id", - ["metadata.version"] = "metadata.version", - ["metadata.product.name"] = "metadata.product.name", - ["metadata.product.vendor_name"] = "metadata.product.vendor_name", - ["dataSource.vendor"] = "dataSource.vendor", - ["dataSource.name"] = "dataSource.name", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["cloudProvider"] = "cloud.provider", - ["accountTypeId"] = "cloud.account.type_id", - ["cloudAccountType"] = "cloud.account.type", - ["message"] = "message", - } -end - --- Finding types constants -local ec2_finding_types = { - ["Backdoor:EC2/C&CActivity.B"] = true, - ["Backdoor:EC2/C&CActivity.B!DNS"] = true, - ["Backdoor:EC2/DenialOfService.Dns"] = true, - ["Backdoor:EC2/DenialOfService.Tcp"] = true, - ["Backdoor:EC2/DenialOfService.Udp"] = true, - ["Backdoor:EC2/DenialOfService.UdpOnTcpPorts"] = true, - ["Backdoor:EC2/DenialOfService.UnusualProtocol"] = true, - ["Backdoor:EC2/Spambot"] = true, - ["Behavior:EC2/NetworkPortUnusual"] = true, - ["Behavior:EC2/TrafficVolumeUnusual"] = true, - ["CryptoCurrency:EC2/BitcoinTool.B"] = true, - ["CryptoCurrency:EC2/BitcoinTool.B!DNS"] = true, - ["DefenseEvasion:EC2/UnusualDNSResolver"] = true, - ["DefenseEvasion:EC2/UnusualDoHActivity"] = true, - ["DefenseEvasion:EC2/UnusualDoTActivity"] = true, - ["Impact:EC2/AbusedDomainRequest.Reputation"] = true, - ["Impact:EC2/BitcoinDomainRequest.Reputation"] = true, - ["Impact:EC2/MaliciousDomainRequest.Reputation"] = true, - ["Impact:EC2/PortSweep"] = true, - ["Impact:EC2/SuspiciousDomainRequest.Reputation"] = true, - ["Impact:EC2/WinRMBruteForce"] = true, - ["Recon:EC2/PortProbeEMRUnprotectedPort"] = true, - ["Recon:EC2/PortProbeUnprotectedPort"] = true, - ["Recon:EC2/Portscan"] = true, - ["Trojan:EC2/BlackholeTraffic"] = true, - ["Trojan:EC2/BlackholeTraffic!DNS"] = true, - ["Trojan:EC2/DGADomainRequest.B"] = true, - ["Trojan:EC2/DGADomainRequest.C!DNS"] = true, - ["Trojan:EC2/DNSDataExfiltration"] = true, - ["Trojan:EC2/DriveBySourceTraffic!DNS"] = true, - ["Trojan:EC2/DropPoint"] = true, - ["Trojan:EC2/DropPoint!DNS"] = true, - ["Trojan:EC2/PhishingDomainRequest!DNS"] = true, - ["UnauthorizedAccess:EC2/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:EC2/MetadataDNSRebind"] = true, - ["UnauthorizedAccess:EC2/RDPBruteForce"] = true, - ["UnauthorizedAccess:EC2/SSHBruteForce"] = true, - ["UnauthorizedAccess:EC2/TorClient"] = true, - ["UnauthorizedAccess:EC2/TorRelay"] = true -} - -local runtime_monitoring_finding_types = { - ["CryptoCurrency:Runtime/BitcoinTool.B"] = true, - ["Backdoor:Runtime/C&CActivity.B"] = true, - ["UnauthorizedAccess:Runtime/TorRelay"] = true, - ["UnauthorizedAccess:Runtime/TorClient"] = true, - ["Trojan:Runtime/BlackholeTraffic"] = true, - ["Trojan:Runtime/DropPoint"] = true, - ["CryptoCurrency:Runtime/BitcoinTool.B!DNS"] = true, - ["Backdoor:Runtime/C&CActivity.B!DNS"] = true, - ["Trojan:Runtime/BlackholeTraffic!DNS"] = true, - ["Trojan:Runtime/DropPoint!DNS"] = true, - ["Trojan:Runtime/DGADomainRequest.C!DNS"] = true, - ["Trojan:Runtime/DriveBySourceTraffic!DNS"] = true, - ["Trojan:Runtime/PhishingDomainRequest!DNS"] = true, - ["Impact:Runtime/AbusedDomainRequest.Reputation"] = true, - ["Impact:Runtime/BitcoinDomainRequest.Reputation"] = true, - ["Impact:Runtime/MaliciousDomainRequest.Reputation"] = true, - ["Impact:Runtime/SuspiciousDomainRequest.Reputation"] = true, - ["UnauthorizedAccess:Runtime/MetadataDNSRebind"] = true, - ["Execution:Runtime/NewBinaryExecuted"] = true, - ["PrivilegeEscalation:Runtime/DockerSocketAccessed"] = true, - ["PrivilegeEscalation:Runtime/RuncContainerEscape"] = true, - ["PrivilegeEscalation:Runtime/CGroupsReleaseAgentModified"] = true, - ["DefenseEvasion:Runtime/ProcessInjection.Proc"] = true, - ["DefenseEvasion:Runtime/ProcessInjection.Ptrace"] = true, - ["DefenseEvasion:Runtime/ProcessInjection.VirtualMemoryWrite"] = true, - ["Execution:Runtime/ReverseShell"] = true, - ["DefenseEvasion:Runtime/FilelessExecution"] = true, - ["Impact:Runtime/CryptoMinerExecuted"] = true, - ["Execution:Runtime/NewLibraryLoaded"] = true, - ["PrivilegeEscalation:Runtime/ContainerMountsHostDirectory"] = true, - ["PrivilegeEscalation:Runtime/UserfaultfdUsage"] = true -} - -local iam_finding_types = { - ["CredentialAccess:IAMUser/AnomalousBehavior"] = true, - ["DefenseEvasion:IAMUser/AnomalousBehavior"] = true, - ["Discovery:IAMUser/AnomalousBehavior"] = true, - ["Exfiltration:IAMUser/AnomalousBehavior"] = true, - ["Impact:IAMUser/AnomalousBehavior"] = true, - ["InitialAccess:IAMUser/AnomalousBehavior"] = true, - ["PenTest:IAMUser/KaliLinux"] = true, - ["PenTest:IAMUser/ParrotLinux"] = true, - ["PenTest:IAMUser/PentooLinux"] = true, - ["Persistence:IAMUser/AnomalousBehavior"] = true, - ["Policy:IAMUser/RootCredentialUsage"] = true, - ["PrivilegeEscalation:IAMUser/AnomalousBehavior"] = true, - ["Recon:IAMUser/MaliciousIPCaller"] = true, - ["Recon:IAMUser/MaliciousIPCaller.Custom"] = true, - ["Recon:IAMUser/TorIPCaller"] = true, - ["Stealth:IAMUser/CloudTrailLoggingDisabled"] = true, - ["Stealth:IAMUser/PasswordPolicyChange"] = true, - ["UnauthorizedAccess:IAMUser/ConsoleLoginSuccess.B"] = true, - ["UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.InsideAWS"] = true, - ["UnauthorizedAccess:IAMUser/InstanceCredentialExfiltration.OutsideAWS"] = true, - ["UnauthorizedAccess:IAMUser/MaliciousIPCaller"] = true, - ["UnauthorizedAccess:IAMUser/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:IAMUser/TorIPCaller"] = true -} - -local kubernetes_audit_logs_finding_types = { - ["CredentialAccess:Kubernetes/MaliciousIPCaller"] = true, - ["CredentialAccess:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["CredentialAccess:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["CredentialAccess:Kubernetes/TorIPCaller"] = true, - ["DefenseEvasion:Kubernetes/MaliciousIPCaller"] = true, - ["DefenseEvasion:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["DefenseEvasion:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["DefenseEvasion:Kubernetes/TorIPCaller"] = true, - ["Discovery:Kubernetes/MaliciousIPCaller"] = true, - ["Discovery:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["Discovery:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["Discovery:Kubernetes/TorIPCaller"] = true, - ["Execution:Kubernetes/ExecInKubeSystemPod"] = true, - ["Impact:Kubernetes/MaliciousIPCaller"] = true, - ["Impact:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["Impact:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["Impact:Kubernetes/TorIPCaller"] = true, - ["Persistence:Kubernetes/ContainerWithSensitiveMount"] = true, - ["Persistence:Kubernetes/MaliciousIPCaller"] = true, - ["Persistence:Kubernetes/MaliciousIPCaller.Custom"] = true, - ["Persistence:Kubernetes/SuccessfulAnonymousAccess"] = true, - ["Persistence:Kubernetes/TorIPCaller"] = true, - ["Policy:Kubernetes/AdminAccessToDefaultServiceAccount"] = true, - ["Policy:Kubernetes/AnonymousAccessGranted"] = true, - ["Policy:Kubernetes/ExposedDashboard"] = true, - ["Policy:Kubernetes/KubeflowDashboardExposed"] = true, - ["PrivilegeEscalation:Kubernetes/PrivilegedContainer"] = true, - ["CredentialAccess:Kubernetes/AnomalousBehavior.SecretsAccessed"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleBindingCreated"] = true, - ["Execution:Kubernetes/AnomalousBehavior.ExecInPod"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!PrivilegedContainer"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.WorkloadDeployed!ContainerWithSensitiveMount"] = true, - ["Execution:Kubernetes/AnomalousBehavior.WorkloadDeployed"] = true, - ["PrivilegeEscalation:Kubernetes/AnomalousBehavior.RoleCreated"] = true, - ["Discovery:Kubernetes/AnomalousBehavior.PermissionChecked"] = true -} - -local lambda_protection_finding_types = { - ["Backdoor:Lambda/C&CActivity.B"] = true, - ["CryptoCurrency:Lambda/BitcoinTool.B"] = true, - ["Trojan:Lambda/BlackholeTraffic"] = true, - ["Trojan:Lambda/DropPoint"] = true, - ["UnauthorizedAccess:Lambda/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:Lambda/TorClient"] = true, - ["UnauthorizedAccess:Lambda/TorRelay"] = true -} - -local malware_protection_finding_types = { - ["Execution:EC2/MaliciousFile"] = true, - ["Execution:ECS/MaliciousFile"] = true, - ["Execution:Kubernetes/MaliciousFile"] = true, - ["Execution:Container/MaliciousFile"] = true, - ["Execution:EC2/SuspiciousFile"] = true, - ["Execution:ECS/SuspiciousFile"] = true, - ["Execution:Kubernetes/SuspiciousFile"] = true, - ["Execution:Container/SuspiciousFile"] = true -} - -local rds_protection_finding_types = { - ["CredentialAccess:RDS/AnomalousBehavior.SuccessfulLogin"] = true, - ["CredentialAccess:RDS/AnomalousBehavior.FailedLogin"] = true, - ["CredentialAccess:RDS/AnomalousBehavior.SuccessfulBruteForce"] = true, - ["CredentialAccess:RDS/MaliciousIPCaller.SuccessfulLogin"] = true, - ["CredentialAccess:RDS/MaliciousIPCaller.FailedLogin"] = true, - ["Discovery:RDS/MaliciousIPCaller"] = true, - ["CredentialAccess:RDS/TorIPCaller.SuccessfulLogin"] = true, - ["CredentialAccess:RDS/TorIPCaller.FailedLogin"] = true, - ["Discovery:RDS/TorIPCaller"] = true -} - -local s3_finding_types = { - ["Discovery:S3/AnomalousBehavior"] = true, - ["Discovery:S3/MaliciousIPCaller"] = true, - ["Discovery:S3/MaliciousIPCaller.Custom"] = true, - ["Discovery:S3/TorIPCaller"] = true, - ["Exfiltration:S3/AnomalousBehavior"] = true, - ["Exfiltration:S3/MaliciousIPCaller"] = true, - ["Impact:S3/AnomalousBehavior.Delete"] = true, - ["Impact:S3/AnomalousBehavior.Permission"] = true, - ["Impact:S3/AnomalousBehavior.Write"] = true, - ["Impact:S3/MaliciousIPCaller"] = true, - ["PenTest:S3/KaliLinux"] = true, - ["PenTest:S3/ParrotLinux"] = true, - ["PenTest:S3/PentooLinux"] = true, - ["Policy:S3/AccountBlockPublicAccessDisabled"] = true, - ["Policy:S3/BucketAnonymousAccessGranted"] = true, - ["Policy:S3/BucketBlockPublicAccessDisabled"] = true, - ["Policy:S3/BucketPublicAccessGranted"] = true, - ["Stealth:S3/ServerAccessLoggingDisabled"] = true, - ["UnauthorizedAccess:S3/MaliciousIPCaller.Custom"] = true, - ["UnauthorizedAccess:S3/TorIPCaller"] = true -} - - --- Per-category helpers (mirroring Python OCSFMapperHelper) -local function ec2_findings(event, site_id) - event["class_uid"] = 4001 - event["class_name"] = "Network Activity" - event["category_uid"] = 4 - event["category_name"] = "Network Activity" - event["activity_id"] = "99" - event["activity_name"] = "EC2 finding types" - event["type_uid"] = 400199 - event["type_name"] = "EC2 finding types" - event["eventId"] = "EC2 finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, ec2_findings_ocsf_mapping()) - -- enrich with fields that were set directly on event - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function runtime_monitoring_findings(event, site_id) - event["class_uid"] = 6003 - event["class_name"] = "API Activity" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "EKS Runtime Monitoring finding types" - event["type_uid"] = 600399 - event["type_name"] = "EKS Runtime Monitoring finding types" - event["eventId"] = "EKS Runtime Monitoring finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, runtime_monitoring_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function iam_findings(event, site_id) - event["class_uid"] = 3004 - event["class_name"] = "Entity Management" - event["category_uid"] = 3 - event["category_name"] = "Identity & Access Management" - event["activity_id"] = "99" - event["activity_name"] = "IAM finding types" - event["type_uid"] = 300499 - event["type_name"] = "IAM finding types" - event["eventId"] = "IAM finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, iam_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function kubernetes_audit_log_findings(event, site_id) - event["class_uid"] = 6003 - event["class_name"] = "API Activity" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "Kubernetes Audit Logs finding types" - event["type_uid"] = 600399 - event["type_name"] = "Kubernetes Audit Logs finding types" - event["eventId"] = "Kubernetes Audit Logs finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, kubernetes_audit_log_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function lambda_findings(event, site_id) - event["class_uid"] = 6002 - event["class_name"] = "Application Lifecycle" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "Lambda Protection finding types" - event["type_uid"] = 600299 - event["type_name"] = "Lambda Protection finding types" - event["eventId"] = "Lambda Protection finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, lambda_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function malware_findings(event, site_id) - event["class_uid"] = 2001 - event["class_name"] = "Security Finding" - event["category_uid"] = 2 - event["category_name"] = "Findings" - event["activity_id"] = "99" - event["activity_name"] = "Malware Protection finding types" - event["type_uid"] = 200199 - event["type_name"] = "Malware Protection finding types" - event["eventId"] = "Malware Protection finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, malware_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - return flat -end - -local function rds_findings(event, site_id) - event["class_uid"] = 3002 - event["class_name"] = "Authentication" - event["category_uid"] = 3 - event["category_name"] = "Identity & Access Management" - event["activity_id"] = "99" - event["activity_name"] = "RDS Protection finding types" - event["type_uid"] = 300299 - event["type_name"] = "RDS Protection finding types" - event["eventId"] = "RDS Protection finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, rds_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function s3_findings(event, site_id) - event["class_uid"] = 6003 - event["class_name"] = "API Activity" - event["category_uid"] = 6 - event["category_name"] = "Application Activity" - event["activity_id"] = "99" - event["activity_name"] = "S3 finding types" - event["type_uid"] = 600399 - event["type_name"] = "S3 finding types" - event["eventId"] = "S3 finding types" - common_mapping(event, site_id) - local flat = apply_mapping(event, s3_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - coordinates_mapping(flat) - return flat -end - -local function unknown_findings(event, site_id) - event["class_uid"] = 0 - event["class_name"] = "Base Event" - event["category_uid"] = 0 - event["category_name"] = "Uncategorized" - event["activity_id"] = 0 - event["activity_name"] = "Unknown" - event["type_uid"] = 0 - event["type_name"] = "Unknown" - event["eventId"] = "Unknown" - common_mapping(event, site_id) - local flat = apply_mapping(event, unknown_findings_ocsf_mapping()) - flat["activity_name"] = event["activity_name"] - flat["activity_id"] = event["activity_id"] - flat["class_uid"] = event["class_uid"] - flat["class_name"] = event["class_name"] - flat["category_uid"] = event["category_uid"] - flat["category_name"] = event["category_name"] - flat["type_uid"] = event["type_uid"] - flat["type_name"] = event["type_name"] - flat["severity_id"] = event["severity_id"] - return flat -end - --- Dispatcher. family one of: "ec2","runtime","iam","kubernetes","lambda","malware","rds","s3","unknown" -local function processGuardDutyEvent(event, site_id, family) - local fam = tostring(family or "unknown"):lower() - if fam == "ec2" then return ec2_findings(deepCopy(event), site_id) end - if fam == "runtime" or fam == "eks_runtime" or fam == "runtime_monitoring" then return runtime_monitoring_findings(deepCopy(event), site_id) end - if fam == "iam" then return iam_findings(deepCopy(event), site_id) end - if fam == "kubernetes" or fam == "k8s" then return kubernetes_audit_log_findings(deepCopy(event), site_id) end - if fam == "lambda" then return lambda_findings(deepCopy(event), site_id) end - if fam == "malware" then return malware_findings(deepCopy(event), site_id) end - if fam == "rds" then return rds_findings(deepCopy(event), site_id) end - if fam == "s3" then return s3_findings(deepCopy(event), site_id) end - return unknown_findings(deepCopy(event), site_id) -end - --- Best-effort automatic family detection from GuardDuty event structure -local function detect_family(event) - local type = getByPath(event, {"type"}) - if ec2_finding_types[type] then return "ec2" end - if runtime_monitoring_finding_types[type] then return "runtime" end - if iam_finding_types[type] then return "iam" end - if kubernetes_audit_logs_finding_types[type] then return "kubernetes" end - if lambda_protection_finding_types[type] then return "lambda" end - if malware_protection_finding_types[type] then return "malware" end - if rds_protection_finding_types[type] then return "rds" end - if s3_finding_types[type] then return "s3" end - return "unknown" -end - --- Helper: Encode Lua table to JSON string with field ordering -local function encodeJson(obj, fieldOrder, key) - if obj == nil then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray then - local items = {} - for i = 1, maxIndex do - table.insert(items, encodeJson(obj[i], fieldOrder, key)) - end - return "[" .. table.concat(items, ",") .. "]" - else - local items = {} - local fieldOrdering = fieldOrder[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrdering) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldOrder, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - -- Phase 2: remaining keys (not in fieldOrder) - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrdering) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, fieldOrder, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - return "{" .. table.concat(items, ",") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local function convertUtcToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - -local IGNORE_KEYS = { - _ob = true, - timestamp = true, -- Ignore timestamp as we use start_time/end_time -} - --- Global entry point expected by transform runners -function processEvent(event) - local site_id = nil - local site_tbl = event and event["site"] - if type(site_tbl) == "table" and site_tbl["id"] then - site_id = site_tbl["id"] - elseif event and event["site_id"] then - site_id = event["site_id"] - end - local fam = detect_family(event or {}) - local copy = deepCopy(event, IGNORE_KEYS) - local result = processGuardDutyEvent(copy, site_id, fam) - result.time = convertUtcToMilliseconds(copy["createdAt"]) - result.message = encodeJson(copy, get_msg_field_ordering(fam), "message") - return result -end - diff --git a/pipelines/community/transform_ocsf/darktrace/darktrace.json b/pipelines/community/transform_ocsf/darktrace/darktrace.json deleted file mode 100644 index d9bd808..0000000 --- a/pipelines/community/transform_ocsf/darktrace/darktrace.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Darktrace", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Darktrace. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "darktrace", - "source_name": "darktrace", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "darktrace-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- Darktrace 1.1.0 OCSF Serializer\n\nlocal FEATURES = {\n CLEANUP_EMPTY_NULL = true,\n}\n\nlocal COMMON_MAPPING = {\n {source = \"action\", target = \"action\"},\n {source = \"action_id\", target = \"action_id\"},\n {source = \"category_uid\", target = \"category_uid\"},\n {source = \"category_name\", target = \"category_name\"},\n {source = \"class_uid\", target = \"class_uid\"},\n {source = \"class_name\", target = \"class_name\"},\n {source = \"activity_id\", target = \"activity_id\"},\n {source = \"activity_name\", target = \"activity_name\"},\n {source = \"type_uid\", target = \"type_uid\"},\n {source = \"type_name\", target = \"type_name\"},\n {source = \"severity_id\", target = \"severity_id\"},\n {source = \"metadata.product.vendor_name\", target = \"metadata.product.vendor_name\"},\n {source = \"metadata.product.name\", target = \"metadata.product.name\"},\n {source = \"metadata.version\", target = \"metadata.version\"},\n {source = \"metadata.original_time\", target = \"metadata.original_time\"},\n {source = \"dataSource.category\", target = \"dataSource.category\"},\n {source = \"dataSource.name\", target = \"dataSource.name\"},\n {source = \"dataSource.vendor\", target = \"dataSource.vendor\"},\n {source = \"site.id\", target = \"site.id\"},\n {source = \"message\", target = \"message\"},\n {source = \"user.type_id\", target = \"user.type_id\"},\n {source = \"observables\", target = \"observables\"}\n}\n\nlocal GROUPS_LOGS_MAPPING = {\n {source = \"id\", target = \"metadata.uid\"},\n {source = \"mitreTactics\", target = \"attacks.tactics\"},\n {source = \"category\", target = \"severity\"},\n {source = \"start\", target = \"start_time\"},\n {source = \"start_time\", target = \"time\"},\n {source = \"end\", target = \"end_time\"},\n {source = \"finding_info.uid\", target = \"finding_info.uid\"},\n {source = \"finding_info.first_seen_time\", target = \"finding_info.first_seen_time\"},\n {source = \"finding_info.title\", target = \"finding_info.title\"},\n {source = \"finding_info.product_uid\", target = \"finding_info.product_uid\"},\n {source = \"finding_info.desc\", target = \"finding_info.desc\"}\n}\n\nlocal INCIDENTS_LOGS_MAPPING = {\n {source = \"createdAt\", target = \"finding_info.created_time\"},\n {source = \"mitreTactics\", target = \"attacks.tactics\"},\n {source = \"start_time\", target = \"start_time\"},\n {source = \"end_time\", target = \"end_time\"},\n {source = \"device.namespace_pid\", target = \"device.namespace_pid\"},\n {source = \"device.hostname\", target = \"device.hostname\"},\n {source = \"device.ip\", target = \"device.ip\"},\n {source = \"device.mac\", target = \"device.mac\"},\n {source = \"device.subnet\", target = \"device.subnet\"},\n {source = \"device.uid\", target = \"device.uid\"},\n {source = \"device.subnet_uid\", target = \"device.subnet_uid\"},\n {source = \"device.type_id\", target = \"device.type_id\"},\n {source = \"authorizations\", target = \"authorizations\"},\n {source = \"risk_score\", target = \"risk_score\"},\n {source = \"id\", target = \"finding_info.uid\"},\n {source = \"title\", target = \"finding_info.title\"},\n {source = \"finding_info.first_seen_time\", target = \"finding_info.first_seen_time\"},\n {source = \"finding_info.product_uid\", target = \"finding_info.product_uid\"},\n {source = \"summary\", target = \"finding_info.desc\"},\n {source = \"severity\", target = \"severity\"},\n {source = \"metadata.original_time\", target = \"time\"}\n}\n\nlocal MODEL_BREACHES_LOGS_MAPPING = {\n {source = \"authorizations\", target = \"authorizations\"},\n {source = \"time\", target = \"metadata.original_time\"},\n {source = \"creationTime\", target = \"finding_info.created_time\"},\n {source = \"resources\", target = \"resources\"},\n {source = \"resources_result\", target = \"resources_result\"},\n {source = \"severity\", target = \"severity\"},\n {source = \"device.uid\", target = \"device.uid\"},\n {source = \"device.type_id\", target = \"device.type_id\"}\n}\n\nlocal STATUS_LOGS_MAPPING = {\n {source = \"ipAddress\", target = \"device.ip\"},\n {source = \"hostname\", target = \"device.hostname\"},\n {source = \"uuid\", target = \"device.uid\"},\n {source = \"device.type_id\", target = \"device.type_id\"},\n {source = \"device.type\", target = \"device.type\"},\n {source = \"device.subnet_uid\", target = \"device.subnet_uid\"},\n {source = \"device.network_interfaces\", target = \"device.network_interfaces\"},\n {source = \"severity\", target = \"severity\"}\n}\n\nlocal function convert_to_epoch_milliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+) (%d+):(%d+)\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\n-- Constant Fields\n\nlocal function get_group_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"uid\"] = event[\"id\"],\n [\"original_time\"] = type(event[\"start\"]) == \"number\" and event[\"start\"] or convert_to_epoch_milliseconds(event[\"start\"] or \"\"),\n },\n [\"action\"] = \"Other\",\n [\"action_id\"] = 99,\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Create\",\n [\"event.type\"] = \"Create\",\n [\"category_name\"] = \"Findings\",\n [\"category_uid\"] = 2,\n [\"class_name\"] = \"Detection Finding\",\n [\"class_uid\"] = 2004,\n [\"type_name\"] = \"Detection Finding: Create\",\n [\"type_uid\"] = 200401,\n [\"severity_id\"] = 99,\n [\"time\"] = type(event[\"start\"]) == \"number\" and event[\"start\"] or convert_to_epoch_milliseconds(event[\"start\"] or \"\")\n }\n\n local incident_events = event[\"incidentEvents\"] or {}\n -- Check if list is not empty (Lua tables are 1-indexed)\n if #incident_events > 0 then\n local incident = incident_events[1]\n fields[\"finding_info\"] = {\n [\"uid\"] = incident[\"uuid\"],\n [\"first_seen_time\"] = incident[\"start\"],\n [\"title\"] = incident[\"title\"],\n [\"product_uid\"] = incident[\"triggerDid\"],\n }\n end\n\n local observables_array = {}\n for _, incident in ipairs(incident_events) do\n -- Resource UID observable object\n local uuid_val = incident[\"uuid\"] or \"\"\n if uuid_val ~= \"\" then\n table.insert(observables_array, {type_id = 10, type = \"Resource UID\", name = \"finding_info.uid\", value = uuid_val})\n end\n\n -- Process observable object\n local title_val = incident[\"title\"] or \"\"\n if title_val ~= \"\" then\n table.insert(observables_array, {type_id = 25, type = \"Process\", name = \"finding_info.title\", value = title_val})\n end\n end\n\n if #observables_array > 0 then\n fields[\"observables\"] = observables_array\n end\n\n return fields\nend\n\nlocal function get_incidents_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"original_time\"] = type(event[\"relatedBreaches\"][1][\"timestamp\"]) == \"number\" and event[\"relatedBreaches\"][1][\"timestamp\"] or convert_to_epoch_milliseconds(event[\"relatedBreaches\"][1][\"timestamp\"] or \"\")\n },\n [\"action\"] = \"Other\",\n [\"action_id\"] = 99,\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Create\",\n [\"event.type\"] = \"Create\",\n [\"category_name\"] = \"Findings\",\n [\"category_uid\"] = 2,\n [\"class_name\"] = \"Detection Finding\",\n [\"class_uid\"] = 2004,\n [\"type_name\"] = \"Detection Finding: Create\",\n [\"type_uid\"] = 200401,\n [\"severity_id\"] = 99,\n [\"time\"] = type(event[\"relatedBreaches\"][1][\"timestamp\"]) == \"number\" and event[\"relatedBreaches\"][1][\"timestamp\"] or convert_to_epoch_milliseconds(event[\"relatedBreaches\"][1][\"timestamp\"] or \"\")\n }\n\n local periods = event[\"periods\"] or {}\n if #periods > 0 then\n fields[\"start_time\"] = periods[1][\"start\"]\n fields[\"end_time\"] = periods[1][\"end\"]\n end\n\n local related_breaches = event[\"relatedBreaches\"] or {}\n if #related_breaches > 0 then\n fields[\"authorizations\"] = {\n {\n [\"policy\"] = {\n [\"name\"] = related_breaches[1][\"modelName\"],\n [\"uid\"] = related_breaches[1][\"pbid\"]\n }\n }\n }\n fields[\"risk_score\"] = related_breaches[1][\"threatScore\"]\n end\n\n local breach_devices = event[\"breachDevices\"] or {}\n if #breach_devices > 0 then\n fields[\"device\"] = {\n [\"namespace_pid\"] = breach_devices[1][\"identifier\"],\n [\"hostname\"] = breach_devices[1][\"hostname\"],\n [\"ip\"] = breach_devices[1][\"ip\"],\n [\"mac\"] = breach_devices[1][\"mac\"],\n [\"subnet\"] = breach_devices[1][\"subnet\"],\n [\"uid\"] = breach_devices[1][\"did\"],\n [\"subnet_uid\"] = breach_devices[1][\"sid\"],\n [\"type_id\"] = 0\n }\n end\n\n local observables_array = {}\n for _, incident in ipairs(breach_devices) do\n -- Hostname observable object\n local hostname_val = incident[\"hostname\"] or \"\"\n if hostname_val ~= \"\" then\n local hostname_obj = {}\n hostname_obj[\"type_id\"] = 1\n hostname_obj[\"type\"] = \"Hostname\"\n hostname_obj[\"name\"] = \"device.hostname\"\n hostname_obj[\"value\"] = hostname_val\n table.insert(observables_array, hostname_obj)\n end\n\n -- IP Address observable object\n local ip_val = incident[\"ip\"] or \"\"\n if ip_val ~= \"\" then\n local ip_obj = {}\n ip_obj[\"type_id\"] = 2\n ip_obj[\"type\"] = \"IP Address\"\n ip_obj[\"name\"] = \"device.ip\"\n ip_obj[\"value\"] = ip_val\n table.insert(observables_array, ip_obj)\n end\n\n -- MAC Address observable object\n local mac_val = incident[\"mac\"] or \"\"\n if mac_val ~= \"\" then\n local mac_obj = {}\n mac_obj[\"type_id\"] = 3\n mac_obj[\"type\"] = \"MAC Address\"\n mac_obj[\"name\"] = \"device.mac\"\n mac_obj[\"value\"] = mac_val\n table.insert(observables_array, mac_obj)\n end\n end\n\n if #observables_array > 0 then\n fields[\"observables\"] = observables_array\n end\n\n return fields\nend\n\nlocal function get_modelbreaches_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"original_time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\")\n },\n [\"action\"] = \"Other\",\n [\"action_id\"] = 99,\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Create\",\n [\"event.type\"] = \"Create\",\n [\"category_name\"] = \"Findings\",\n [\"category_uid\"] = 2,\n [\"class_name\"] = \"Detection Finding\",\n [\"class_uid\"] = 2004,\n [\"type_name\"] = \"Detection Finding: Create\",\n [\"type_uid\"] = 200401,\n [\"severity_id\"] = 99,\n [\"time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\")\n }\n\n fields[\"authorizations\"] = {\n {\n [\"policy\"] = {\n [\"uid\"] = event[\"pbid\"]\n }\n }\n }\n\n local model = event[\"model\"] or {}\n local model_then = model[\"then\"]\n -- In Lua, empty tables are true, so check if not nil.\n -- If 'then' might be an empty table {}, you might need next(model_then) check, \n -- but usually API responses omit keys or return nil if missing.\n if model_then then\n fields[\"resources\"] = {\n {\n [\"name\"] = model_then[\"name\"],\n [\"uid\"] = model_then[\"uuid\"],\n [\"version\"] = model_then[\"version\"],\n [\"data\"] = model_then\n }\n }\n end\n\n local model_now = model[\"now\"]\n -- Note: Original Python code had a bug: 'if model_then:' check for 'resources_result'\n -- I assumed you wanted 'if model_now:' here based on context.\n -- If you want strict translation of the bug, change 'model_now' to 'model_then' in the if condition.\n if model_now then \n fields[\"resources_result\"] = {\n {\n [\"name\"] = model_now[\"name\"],\n [\"uid\"] = model_now[\"uuid\"],\n [\"version\"] = model_now[\"version\"],\n [\"data\"] = model_now\n }\n }\n end\n\n local device = event[\"device\"] or {}\n local device_did = device[\"did\"]\n if device_did then\n fields[\"device\"] = {\n [\"uid\"] = device_did,\n [\"type_id\"] = 0\n }\n end\n\n return fields\nend\n\nlocal function get_status_logs_computed_fields(event)\n local fields = {\n [\"dataSource\"] = {\n [\"category\"] = \"security\",\n [\"name\"] = \"Darktrace\",\n [\"vendor\"] = \"Darktrace\",\n },\n [\"metadata\"] = {\n [\"product\"] = {\n [\"name\"] = \"Darktrace\",\n [\"vendor_name\"] = \"Darktrace\",\n },\n [\"version\"] = \"1.1.0\",\n [\"original_time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\")\n },\n [\"device\"] = {\n [\"type_id\"] = 99,\n [\"type\"] = event[\"type\"],\n [\"hostname\"] = event[\"hostname\"],\n [\"ip\"] = event[\"ipAddress\"],\n [\"network_interfaces\"] = {\n {\n [\"ip\"] = event[\"networkInterfacesAddress_eth0\"],\n [\"type_id\"] = 1,\n [\"type\"] = \"Wired\"\n }\n }\n },\n [\"time\"] = type(event[\"time\"]) == \"number\" and event[\"time\"] or convert_to_epoch_milliseconds(event[\"time\"] or \"\"),\n [\"activity_id\"] = 1,\n [\"activity_name\"] = \"Log\",\n [\"event.type\"] = \"Log\",\n [\"category_name\"] = \"Discovery\",\n [\"category_uid\"] = 5,\n [\"class_name\"] = \"Device Config State\",\n [\"class_uid\"] = 5002,\n [\"type_name\"] = \"Device Config State: Log\",\n [\"type_uid\"] = 500201,\n [\"severity_id\"] = 99,\n }\n \n -- Initialize device table\n\n local subnet_data = event[\"subnetData\"] or {}\n if #subnet_data >= 1 then\n local sid = subnet_data[1][\"sid\"]\n fields[\"device\"][\"subnet_uid\"] = sid\n end\n\n fields[\"observables\"] = {}\n \n local hostname = event[\"hostname\"]\n if hostname and hostname ~= \"\" then\n table.insert(fields[\"observables\"], {\n [\"type_id\"] = 1,\n [\"type\"] = \"Hostname\",\n [\"name\"] = \"device.hostname\",\n [\"value\"] = hostname,\n })\n end\n\n local ip_address = event[\"ipAddress\"]\n if ip_address and ip_address ~= \"\" then\n table.insert(fields[\"observables\"], {\n [\"type_id\"] = 2,\n [\"type\"] = \"IP Address\",\n [\"name\"] = \"device.ip\",\n [\"value\"] = ip_address,\n })\n end\n\n return fields\nend\n\n\n\n\nlocal function getConstantFields(event_type, event) \n if event_type == \"aianalyst/groups\" then\n return get_group_logs_computed_fields(event)\n elseif event_type == \"aianalyst/incidentevents\" then\n return get_incidents_logs_computed_fields(event)\n elseif event_type == \"modelbreaches\" then\n return get_modelbreaches_logs_computed_fields(event)\n elseif event_type == \"status\" then\n return get_status_logs_computed_fields(event)\n end\nend\n\nlocal IGNORE_KEYS = {\n _ob = true,\n _darktrace_event_type = true,\n _darktrace_query_time = true,\n _darktrace_start_time = true,\n _darktrace_end_time = true,\n timestamp = true\n}\n\n\n-- Utility functions\nlocal function split(str, delimiter)\n if not str or str == \"\" then\n return {}\n end\n local result = {}\n for token in string.gmatch(str, \"([^\" .. delimiter .. \"]+)\") do\n table.insert(result, token)\n end\n return result\nend\n\nlocal function getValueByPath(obj, path)\n local current = obj\n for _, part in ipairs(split(path, \".\")) do\n if type(current) ~= \"table\" then\n return nil\n end\n local key = tonumber(part) or part\n current = current[key]\n end\n return current\nend\n\nlocal function setValueByPath(obj, path, value)\n local parts = split(path, \".\")\n local current = obj\n for i = 1, #parts - 1 do\n local key = tonumber(parts[i]) or parts[i]\n if current[key] == nil or type(current[key]) ~= \"table\" then\n current[key] = {}\n end\n current = current[key]\n end\n local last = parts[#parts]\n if value == nil then\n current[last] = nil\n else\n current[last] = value\n end\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n return value\n end\n local copy = {}\n for k, v in pairs(value) do\n if not (ignoreKeys and ignoreKeys[k]) then\n copy[k] = deepCopy(v, ignoreKeys)\n end\n end\n return copy\nend\n\n-- Helper: Encode Lua table to JSON string with field ordering\nlocal function encodeJson(obj, fieldOrder, key)\n if obj == nil then\n return \"null\"\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n if isArray then\n local items = {}\n for i = 1, maxIndex do\n table.insert(items, encodeJson(obj[i], fieldOrder, key))\n end\n return \"[\" .. table.concat(items, \",\") .. \"]\"\n else\n local items = {}\n local fieldOrdering = fieldOrder[key] or {}\n \n -- Phase 1: ordered keys\n for _, fieldName in ipairs(fieldOrdering) do\n local v = obj[fieldName]\n if v ~= nil then\n local encoded = encodeJson(v, fieldOrder, fieldName)\n if encoded ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n end\n end\n end\n \n -- Phase 2: remaining keys (not in fieldOrder)\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrdering) do\n if k == fieldName then\n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local encoded = encodeJson(v, fieldOrder, keyStr)\n if encoded ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n end\n end\n end\n \n return \"{\" .. table.concat(items, \",\") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal GROUPS_FIELD_ORDER = {\n message = {\n \"id\", \"active\", \"acknowledged\", \"pinned\", \"userTriggered\", \"externalTriggered\", \n \"previousIds\", \"incidentEvents\", \"mitreTactics\", \"devices\", \"initialDevices\", \n \"category\", \"groupScore\", \"start\", \"end\", \"edges\"\n },\n incidentEvents = {\n \"uuid\", \"start\", \"title\", \"triggerDid\", \"visible\"\n },\n edges = {\n \"isAction\", \"source\", \"target\", \"start\", \"incidentEvent\", \"description\", \"details\"\n },\n source = {\n \"nodeType\", \"value\"\n },\n target = {\n \"nodeType\", \"value\"\n }\n}\n\nlocal INCIDENT_FIELD_ORDER = {\n message = {\n \"summariser\", \"acknowledged\", \"pinned\", \"createdAt\", \"attackPhases\", \"mitreTactics\",\n \"title\", \"id\", \"children\", \"category\", \"currentGroup\", \"groupCategory\", \"groupScore\",\n \"groupPreviousGroups\", \"activityId\", \"groupingIds\", \"groupByActivity\", \"userTriggered\",\n \"externalTriggered\", \"aiaScore\", \"summary\", \"periods\", \"breachDevices\", \"relatedBreaches\",\n \"details\"\n },\n periods = {\n \"start\", \"end\"\n },\n breachDevices = {\n \"identifier\", \"hostname\", \"ip\", \"mac\", \"subnet\", \"did\", \"sid\"\n },\n relatedBreaches = {\n \"modelName\", \"pbid\", \"threatScore\", \"timestamp\"\n },\n details = {\n \"contents\", \"header\"\n },\n contents = {\n \"key\", \"type\", \"values\"\n },\n values = {\n \"identifier\", \"hostname\", \"ip\", \"mac\", \"subnet\", \"did\", \"sid\"\n }\n}\n\nlocal MODEL_BREACH_FIELD_ORDER = {\n message = {\n \"commentCount\", \"pbid\", \"time\", \"creationTime\", \"model\", \n \"triggeredComponents\", \"score\", \"device\"\n },\n model = {\n \"then\", \"now\"\n },\n -- 'then' is a reserved keyword in Lua, so we use bracket notation\n [\"then\"] = {\n \"name\", \"pid\", \"phid\", \"uuid\", \"logic\", \"throttle\", \"sharedEndpoints\", \n \"actions\", \"tags\", \"interval\", \"delay\", \"sequenced\", \"active\", \"modified\", \n \"activeTimes\", \"autoUpdatable\", \"autoUpdate\", \"autoSuppress\", \"description\", \n \"behaviour\", \"defeats\", \"created\", \"edited\", \"message\", \"version\", \n \"priority\", \"category\", \"compliance\"\n },\n now = {\n \"name\", \"pid\", \"phid\", \"uuid\", \"logic\", \"throttle\", \"sharedEndpoints\", \n \"actions\", \"tags\", \"interval\", \"delay\", \"sequenced\", \"active\", \"modified\", \n \"activeTimes\", \"autoUpdatable\", \"autoUpdate\", \"autoSuppress\", \"description\", \n \"behaviour\", \"defeats\", \"created\", \"edited\", \"message\", \"version\", \n \"priority\", \"category\", \"compliance\"\n },\n actions = {\n \"alert\", \"antigena\", \"breach\", \"model\", \"setPriority\", \"setTag\", \"setType\"\n },\n activeTimes = {\n \"devices\", \"tags\", \"type\", \"version\"\n },\n created = { \"by\" },\n edited = { \"by\" },\n triggeredComponents = {\n \"time\", \"cbid\", \"cid\", \"chid\", \"size\", \"threshold\", \"interval\", \n \"logic\", \"version\", \"metric\", \"triggeredFilters\"\n },\n metric = {\n \"mlid\", \"name\", \"label\"\n },\n triggeredFilters = {\n \"cfid\", \"id\", \"filterType\", \"arguments\", \"comparatorType\", \"trigger\"\n },\n arguments = { \"value\" },\n trigger = { \"value\" },\n device = { \"did\" },\n \n -- Logic structures (Shared by model logic and component logic)\n logic = {\n \"data\", \"type\", \"version\"\n },\n -- Recursive data structure for component logic\n data = {\n \"left\", \"operator\", \"right\"\n },\n left = {\n \"left\", \"operator\", \"right\"\n },\n right = {\n \"left\", \"operator\", \"right\"\n }\n}\n\nlocal STATUS_FIELD_ORDER = {\n message = {\n \"excessTraffic\", \"time\", \"installed\", \"mobileAppConfigured\", \"version\", \"ipAddress\",\n \"modelsUpdated\", \"modelPackageVersion\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\",\n \"hostname\", \"inoculation\", \"applianceOSCode\", \"license\", \"saasConnectorLicense\",\n \"antigenaSaasLicense\", \"syslogTLSSHA1Fingerprint\", \"syslogTLSSHA256Fingerprint\",\n \"antigenaNetworkEnabled\", \"antigenaNetworkLicense\", \"antigenaNetworkRunning\",\n \"logIngestionReplicated\", \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\",\n \"logIngestionTypes\", \"logIngestionMatches\", \"licenseCounts\", \"type\", \"diskUtilization\",\n \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\", \"dataQueue\", \"darkflowQueue\",\n \"networkInterfacesState_eth0\", \"networkInterfacesAddress_eth0\",\n \"networkInterfacesReceived_eth0\", \"networkInterfacesTransmitted_eth0\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\", \"bandwidth2WeekPeakString\",\n \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\", \"processedBandwidthAverage\",\n \"processedBandwidthAverageString\", \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"eventsPerMinuteCurrent\", \"probes\", \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\", \"operatingSystems\",\n \"newDevices4Weeks\", \"newDevices7Days\", \"newDevices24Hours\", \"newDevicesHour\",\n \"activeDevices4Weeks\", \"activeDevices7Days\", \"activeDevices24Hours\", \"activeDevicesHour\",\n \"deviceHostnames\", \"deviceMACAddresses\", \"deviceRecentIPChange\", \"models\", \"modelsBreached\",\n \"modelsSuppressed\", \"devicesModeled\", \"recentUnidirectionalConnections\",\n \"mostRecentCOTPTraffic\", \"mostRecentDCE_RPCTraffic\", \"mostRecentDHCPTraffic\",\n \"mostRecentDNSTraffic\", \"mostRecentFTPTraffic\", \"mostRecentGSSAPITraffic\", \"mostRecentH2Traffic\",\n \"mostRecentHTTPTraffic\", \"mostRecentHTTPSTraffic\", \"mostRecentKERBEROSTraffic\",\n \"mostRecentLDAPTraffic\", \"mostRecentNETLOGONTraffic\", \"mostRecentNTLMTraffic\",\n \"mostRecentNTPTraffic\", \"mostRecentRDPTraffic\", \"mostRecentSMBTraffic\", \"mostRecentSOCKSTraffic\",\n \"mostRecentSSHTraffic\", \"mostRecentSSLTraffic\", \"ignoreAnalysisCredentials\",\n \"internalIPRangeList\", \"internalIPRanges\", \"dnsServers\", \"internalDomains\",\n \"internalAndExternalDomains\", \"proxyServers\", \"subnets\", \"subnetData\"\n },\n probes = {\"1\", \"2\", \"3\"},\n [\"1\"] = {\n \"id\", \"mappedId\", \"antigenaNetworkBlockedConnections\", \"configuredServer\", \"version\",\n \"ipAddress\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\", \"metadata\", \"hostname\",\n \"time\", \"installed\", \"kernel\", \"applianceOSCode\", \"syslogTLSSHA1Fingerprint\",\n \"syslogTLSSHA256Fingerprint\", \"antigenaNetworkRunning\", \"logIngestionReplicated\",\n \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\", \"logIngestionDecryption\",\n \"type\", \"diskUtilization\", \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\",\n \"networkInterfacesState_ens5\", \"networkInterfacesAddress_ens5\",\n \"networkInterfacesReceived_ens5\", \"networkInterfacesTransmitted_ens5\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\",\n \"bandwidth2WeekPeakString\", \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\",\n \"processedBandwidthAverage\", \"processedBandwidthAverageString\",\n \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\",\n \"label\", \"error\", \"lastContact\"\n },\n [\"2\"] = {\n \"id\", \"mappedId\", \"antigenaNetworkBlockedConnections\", \"configuredServer\", \"version\",\n \"ipAddress\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\", \"metadata\", \"hostname\",\n \"time\", \"installed\", \"kernel\", \"applianceOSCode\", \"syslogTLSSHA1Fingerprint\",\n \"syslogTLSSHA256Fingerprint\", \"antigenaNetworkRunning\", \"logIngestionReplicated\",\n \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\", \"logIngestionDecryption\",\n \"type\", \"diskUtilization\", \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\",\n \"networkInterfacesState_ens5\", \"networkInterfacesAddress_ens5\",\n \"networkInterfacesReceived_ens5\", \"networkInterfacesTransmitted_ens5\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\",\n \"bandwidth2WeekPeakString\", \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\",\n \"processedBandwidthAverage\", \"processedBandwidthAverageString\",\n \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\",\n \"label\", \"error\", \"lastContact\"\n },\n [\"3\"] = {\n \"id\", \"mappedId\", \"antigenaNetworkBlockedConnections\", \"configuredServer\", \"version\",\n \"ipAddress\", \"bundleVersion\", \"bundleDate\", \"bundleInstalledDate\", \"metadata\", \"hostname\",\n \"time\", \"installed\", \"kernel\", \"applianceOSCode\", \"syslogTLSSHA1Fingerprint\",\n \"syslogTLSSHA256Fingerprint\", \"antigenaNetworkRunning\", \"logIngestionReplicated\",\n \"logIngestionProcessed\", \"logIngestionTCP\", \"logIngestionUDP\", \"logIngestionDecryption\",\n \"type\", \"diskUtilization\", \"uptime\", \"systemUptime\", \"load\", \"cpu\", \"memoryUsed\",\n \"networkInterfacesState_ens5\", \"networkInterfacesAddress_ens5\",\n \"networkInterfacesReceived_ens5\", \"networkInterfacesTransmitted_ens5\",\n \"bandwidthCurrent\", \"bandwidthCurrentString\", \"bandwidthAverage\", \"bandwidthAverageString\",\n \"bandwidth7DayPeak\", \"bandwidth7DayPeakString\", \"bandwidth2WeekPeak\",\n \"bandwidth2WeekPeakString\", \"processedBandwidthCurrent\", \"processedBandwidthCurrentString\",\n \"processedBandwidthAverage\", \"processedBandwidthAverageString\",\n \"processedBandwidth7DayPeak\", \"processedBandwidth7DayPeakString\",\n \"processedBandwidth2WeekPeak\", \"processedBandwidth2WeekPeakString\",\n \"connectionsPerMinuteCurrent\", \"connectionsPerMinuteAverage\",\n \"connectionsPerMinute7DayPeak\", \"connectionsPerMinute2WeekPeak\",\n \"label\", \"error\", \"lastContact\"\n },\n\n antigenaNetworkBlockedConnections = {\n \"attempted\", \"failed\"\n },\n \n subnetData = {\n \"sid\", \"network\", \"devices\", \"clientDevices\", \"mostRecentTraffic\", \"mostRecentDHCP\", \"kerberosQuality\"\n },\n \n eventsPerMinuteCurrent = {\n \"networkConnections\", \"logInputConnections\", \"cSensorConnections\", \"cSensorNotices\",\n \"cSensorDeviceDetails\", \"cSensorModelEvents\", \"networkNotices\", \"networkDeviceDetails\",\n \"networkModelEvents\", \"logInputNotices\", \"logInputDeviceDetails\", \"logInputModelEvents\",\n \"saasNotices\", \"saasModelEvents\"\n },\n \n licenseCounts = {\n \"saas\", \"licenseIPCount\"\n },\n \n saas = {\n \"total\"\n },\n\n -- Dynamic keys; listing those present in the log to ensure order if they appear\n logIngestionTypes = {\n \"TestingDeviceObjects-connectionlogs\", \"TestingDeviceObjects2-connectionlogs\",\n \"TestingDeviceObjects3-connectionlogs\", \"TestingDeviceObjects4-connectionlogs\"\n },\n \n logIngestionMatches = {\n \"TestMatch\", \"TestSrcHostname\", \"TestingDeviceObjects-connectionlogs\"\n }\n}\n\n\nlocal function get_msg_field_ordering(event_type)\n if event_type == \"aianalyst/groups\" then\n return GROUPS_FIELD_ORDER\n elseif event_type == \"aianalyst/incidentevents\" then\n return INCIDENT_FIELD_ORDER\n elseif event_type == \"modelbreaches\" then\n return MODEL_BREACH_FIELD_ORDER\n elseif event_type == \"status\" then\n return STATUS_FIELD_ORDER\n end\nend\n\nlocal function cleanupEmptyNull(obj)\n if type(obj) ~= \"table\" then\n if type(obj) == \"string\" then\n local lower = obj:lower()\n if lower == \"\" or lower == \"null\" then\n return nil\n end\n end\n return obj\n end\n for key, value in pairs(obj) do\n local cleaned = cleanupEmptyNull(value)\n if cleaned == nil then\n obj[key] = nil\n else\n obj[key] = cleaned\n end\n end\n if next(obj) == nil then\n return nil\n end\n return obj\nend\n\nlocal function collectUnmapped(source, target)\n for key, value in pairs(source or {}) do\n if type(value) == \"table\" then\n local nested = {}\n collectUnmapped(value, nested)\n if next(nested) then\n target[key] = nested\n end\n else\n target[key] = value\n end\n end\nend\n\nlocal function getEventType(source)\n if source[\"start\"] then\n return \"aianalyst/groups\"\n elseif source[\"relatedBreaches\"] then\n return \"aianalyst/incidentevents\"\n elseif source[\"time\"] and source[\"triggeredComponents\"] then\n return \"modelbreaches\"\n else\n return \"status\"\n end\nend\n\nfunction processEvent(event)\n if type(event) ~= \"table\" then\n return nil\n end\n\n local MAPPED_FIELDS = {}\n\n local source = deepCopy(event, IGNORE_KEYS)\n\n local event_type = getEventType(source)\n\n local result = {}\n \n for _, mapping in ipairs(COMMON_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n\n if event_type == \"aianalyst/groups\" then\n for _, mapping in ipairs(GROUPS_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n elseif event_type == \"aianalyst/incidentevents\" then\n for _, mapping in ipairs(INCIDENTS_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n elseif event_type == \"modelbreaches\" then\n for _, mapping in ipairs(MODEL_BREACHES_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n elseif event_type == \"status\" then\n for _, mapping in ipairs(STATUS_LOGS_MAPPING) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n end\n MAPPED_FIELDS[mapping.source] = true\n end\n end\n\n for key, value in pairs(getConstantFields(event_type, event)) do\n result[key] = value\n end\n\n -- Remove mapped fields from original event for unmapped collection\n for key, _ in pairs(MAPPED_FIELDS) do\n setValueByPath(event, key, nil)\n end\n \n for key, _ in pairs(IGNORE_KEYS) do\n setValueByPath(event, key, nil)\n end\n\n \n\n local unmapped = {}\n collectUnmapped(event, unmapped)\n unmapped[\"event.type\"] = event_type\n result.unmapped = {}\n if next(unmapped) then\n -- Merge with existing unmapped\n for k, v in pairs(unmapped) do\n result.unmapped[k] = v\n end\n end\n\n result.message = encodeJson(source, get_msg_field_ordering(event_type), \"message\")\n\n if FEATURES.CLEANUP_EMPTY_NULL then\n cleanupEmptyNull(result)\n end\n\n return result\nend", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/darktrace/metadata.yaml b/pipelines/community/transform_ocsf/darktrace/metadata.yaml deleted file mode 100644 index e025630..0000000 --- a/pipelines/community/transform_ocsf/darktrace/metadata.yaml +++ /dev/null @@ -1,51 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Darktrace. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: darktrace - dataSource: Darktrace - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "Syslog" - auth_type: "N/A" - sample_record: "{\n \"time\": 1776656452626,\n \"creationTime\": 1776656333626,\n \"model\": {\n\ - \ \"name\": \"Anomalous File / Internet Facing System File Download\",\n \"description\": \"\ - An internet-facing system has downloaded an unusual file type\",\n \"id\": 861,\n \"version\"\ - : 4,\n \"uuid\": \"789427c6-14d5-4a2b-9e0c-2b0fafb27ed4\"\n },\n \"breachUrl\": \"https://darktrace-07fa459a-0001-01/#modelbreach/71659\"\ - ,\n \"pbid\": 5394290,\n \"score\": 0.722,\n \"device\": {\n \"hostname\": \"PRINTER-971\",\n\ - \ \"ip\": \"10.98.33.252\",\n \"mac\": \"55:de:36:e2:4a:d6\",\n \"type\": \"router\",\n \ - \ \"os\": \"Unknown\"\n },\n \"triggeredComponents\": [\n {\n \"time\": 1776656452626,\n\ - \ \"uid\": \"eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4\",\n \"pid\": 9692,\n \"detail\"\ - : {\n \"fileName\": \"chrome_update.exe\",\n \"fileHash\": \"ea54cb6d1ea944ee9c0a9e70a006c931\"\ - ,\n \"fileSize\": 424534,\n \"downloadSource\": \"http://crypto-miner-4.io/download\"\ - \n }\n },\n {\n \"time\": 1776656452626,\n \"uid\": \"eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4\"\ - ,\n \"pid\": 9692,\n \"detail\": {\n \"connections\": 33,\n \"bytesIn\": 7258485,\n\ - \ \"bytesOut\": 4496939,\n \"duration\": 2974\n }\n }\n ],\n \"commentCount\"\ - : 4,\n \"acknowledged\": true,\n \"category\": \"malware\",\n \"mitreTactics\": [\n \"Execution\"\ - ,\n \"Defense Evasion\"\n ],\n \"tags\": [\n \"darktrace\",\n \"anomaly\",\n \"security\"\ - ,\n \"malware\",\n \"trojan\",\n \"virus\"\n ]\n}" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: darktrace, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/darktrace/sample.json b/pipelines/community/transform_ocsf/darktrace/sample.json deleted file mode 100644 index 89f7fad..0000000 --- a/pipelines/community/transform_ocsf/darktrace/sample.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "time": 1776656452626, - "creationTime": 1776656333626, - "model": { - "name": "Anomalous File / Internet Facing System File Download", - "description": "An internet-facing system has downloaded an unusual file type", - "id": 861, - "version": 4, - "uuid": "789427c6-14d5-4a2b-9e0c-2b0fafb27ed4" - }, - "breachUrl": "https://darktrace-07fa459a-0001-01/#modelbreach/71659", - "pbid": 5394290, - "score": 0.722, - "device": { - "hostname": "PRINTER-971", - "ip": "10.98.33.252", - "mac": "55:de:36:e2:4a:d6", - "type": "router", - "os": "Unknown" - }, - "triggeredComponents": [ - { - "time": 1776656452626, - "uid": "eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4", - "pid": 9692, - "detail": { - "fileName": "chrome_update.exe", - "fileHash": "ea54cb6d1ea944ee9c0a9e70a006c931", - "fileSize": 424534, - "downloadSource": "http://crypto-miner-4.io/download" - } - }, - { - "time": 1776656452626, - "uid": "eb6c21aa-6a55-4a9a-91f5-457fc94ac2d4", - "pid": 9692, - "detail": { - "connections": 33, - "bytesIn": 7258485, - "bytesOut": 4496939, - "duration": 2974 - } - } - ], - "commentCount": 4, - "acknowledged": true, - "category": "malware", - "mitreTactics": [ - "Execution", - "Defense Evasion" - ], - "tags": [ - "darktrace", - "anomaly", - "security", - "malware", - "trojan", - "virus" - ] -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/darktrace/serializer.lua b/pipelines/community/transform_ocsf/darktrace/serializer.lua deleted file mode 100644 index d85962b..0000000 --- a/pipelines/community/transform_ocsf/darktrace/serializer.lua +++ /dev/null @@ -1,987 +0,0 @@ --- Darktrace 1.1.0 OCSF Serializer - -local FEATURES = { - CLEANUP_EMPTY_NULL = true, -} - -local COMMON_MAPPING = { - {source = "action", target = "action"}, - {source = "action_id", target = "action_id"}, - {source = "category_uid", target = "category_uid"}, - {source = "category_name", target = "category_name"}, - {source = "class_uid", target = "class_uid"}, - {source = "class_name", target = "class_name"}, - {source = "activity_id", target = "activity_id"}, - {source = "activity_name", target = "activity_name"}, - {source = "type_uid", target = "type_uid"}, - {source = "type_name", target = "type_name"}, - {source = "severity_id", target = "severity_id"}, - {source = "metadata.product.vendor_name", target = "metadata.product.vendor_name"}, - {source = "metadata.product.name", target = "metadata.product.name"}, - {source = "metadata.version", target = "metadata.version"}, - {source = "metadata.original_time", target = "metadata.original_time"}, - {source = "dataSource.category", target = "dataSource.category"}, - {source = "dataSource.name", target = "dataSource.name"}, - {source = "dataSource.vendor", target = "dataSource.vendor"}, - {source = "site.id", target = "site.id"}, - {source = "message", target = "message"}, - {source = "user.type_id", target = "user.type_id"}, - {source = "observables", target = "observables"} -} - -local GROUPS_LOGS_MAPPING = { - {source = "id", target = "metadata.uid"}, - {source = "mitreTactics", target = "attacks.tactics"}, - {source = "category", target = "severity"}, - {source = "start", target = "start_time"}, - {source = "start_time", target = "time"}, - {source = "end", target = "end_time"}, - {source = "finding_info.uid", target = "finding_info.uid"}, - {source = "finding_info.first_seen_time", target = "finding_info.first_seen_time"}, - {source = "finding_info.title", target = "finding_info.title"}, - {source = "finding_info.product_uid", target = "finding_info.product_uid"}, - {source = "finding_info.desc", target = "finding_info.desc"} -} - -local INCIDENTS_LOGS_MAPPING = { - {source = "createdAt", target = "finding_info.created_time"}, - {source = "mitreTactics", target = "attacks.tactics"}, - {source = "start_time", target = "start_time"}, - {source = "end_time", target = "end_time"}, - {source = "device.namespace_pid", target = "device.namespace_pid"}, - {source = "device.hostname", target = "device.hostname"}, - {source = "device.ip", target = "device.ip"}, - {source = "device.mac", target = "device.mac"}, - {source = "device.subnet", target = "device.subnet"}, - {source = "device.uid", target = "device.uid"}, - {source = "device.subnet_uid", target = "device.subnet_uid"}, - {source = "device.type_id", target = "device.type_id"}, - {source = "authorizations", target = "authorizations"}, - {source = "risk_score", target = "risk_score"}, - {source = "id", target = "finding_info.uid"}, - {source = "title", target = "finding_info.title"}, - {source = "finding_info.first_seen_time", target = "finding_info.first_seen_time"}, - {source = "finding_info.product_uid", target = "finding_info.product_uid"}, - {source = "summary", target = "finding_info.desc"}, - {source = "severity", target = "severity"}, - {source = "metadata.original_time", target = "time"} -} - -local MODEL_BREACHES_LOGS_MAPPING = { - {source = "authorizations", target = "authorizations"}, - {source = "time", target = "metadata.original_time"}, - {source = "creationTime", target = "finding_info.created_time"}, - {source = "resources", target = "resources"}, - {source = "resources_result", target = "resources_result"}, - {source = "severity", target = "severity"}, - {source = "device.uid", target = "device.uid"}, - {source = "device.type_id", target = "device.type_id"} -} - -local STATUS_LOGS_MAPPING = { - {source = "ipAddress", target = "device.ip"}, - {source = "hostname", target = "device.hostname"}, - {source = "uuid", target = "device.uid"}, - {source = "device.type_id", target = "device.type_id"}, - {source = "device.type", target = "device.type"}, - {source = "device.subnet_uid", target = "device.subnet_uid"}, - {source = "device.network_interfaces", target = "device.network_interfaces"}, - {source = "severity", target = "severity"} -} - -local function convert_to_epoch_milliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+) (%d+):(%d+)") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - --- Constant Fields - -local function get_group_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["uid"] = event["id"], - ["original_time"] = type(event["start"]) == "number" and event["start"] or convert_to_epoch_milliseconds(event["start"] or ""), - }, - ["action"] = "Other", - ["action_id"] = 99, - ["activity_id"] = 1, - ["activity_name"] = "Create", - ["event.type"] = "Create", - ["category_name"] = "Findings", - ["category_uid"] = 2, - ["class_name"] = "Detection Finding", - ["class_uid"] = 2004, - ["type_name"] = "Detection Finding: Create", - ["type_uid"] = 200401, - ["severity_id"] = 99, - ["time"] = type(event["start"]) == "number" and event["start"] or convert_to_epoch_milliseconds(event["start"] or "") - } - - local incident_events = event["incidentEvents"] or {} - -- Check if list is not empty (Lua tables are 1-indexed) - if #incident_events > 0 then - local incident = incident_events[1] - fields["finding_info"] = { - ["uid"] = incident["uuid"], - ["first_seen_time"] = incident["start"], - ["title"] = incident["title"], - ["product_uid"] = incident["triggerDid"], - } - end - - local observables_array = {} - for _, incident in ipairs(incident_events) do - -- Resource UID observable object - local uuid_val = incident["uuid"] or "" - if uuid_val ~= "" then - table.insert(observables_array, {type_id = 10, type = "Resource UID", name = "finding_info.uid", value = uuid_val}) - end - - -- Process observable object - local title_val = incident["title"] or "" - if title_val ~= "" then - table.insert(observables_array, {type_id = 25, type = "Process", name = "finding_info.title", value = title_val}) - end - end - - if #observables_array > 0 then - fields["observables"] = observables_array - end - - return fields -end - -local function get_incidents_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["original_time"] = type(event["relatedBreaches"][1]["timestamp"]) == "number" and event["relatedBreaches"][1]["timestamp"] or convert_to_epoch_milliseconds(event["relatedBreaches"][1]["timestamp"] or "") - }, - ["action"] = "Other", - ["action_id"] = 99, - ["activity_id"] = 1, - ["activity_name"] = "Create", - ["event.type"] = "Create", - ["category_name"] = "Findings", - ["category_uid"] = 2, - ["class_name"] = "Detection Finding", - ["class_uid"] = 2004, - ["type_name"] = "Detection Finding: Create", - ["type_uid"] = 200401, - ["severity_id"] = 99, - ["time"] = type(event["relatedBreaches"][1]["timestamp"]) == "number" and event["relatedBreaches"][1]["timestamp"] or convert_to_epoch_milliseconds(event["relatedBreaches"][1]["timestamp"] or "") - } - - local periods = event["periods"] or {} - if #periods > 0 then - fields["start_time"] = periods[1]["start"] - fields["end_time"] = periods[1]["end"] - end - - local related_breaches = event["relatedBreaches"] or {} - if #related_breaches > 0 then - fields["authorizations"] = { - { - ["policy"] = { - ["name"] = related_breaches[1]["modelName"], - ["uid"] = related_breaches[1]["pbid"] - } - } - } - fields["risk_score"] = related_breaches[1]["threatScore"] - end - - local breach_devices = event["breachDevices"] or {} - if #breach_devices > 0 then - fields["device"] = { - ["namespace_pid"] = breach_devices[1]["identifier"], - ["hostname"] = breach_devices[1]["hostname"], - ["ip"] = breach_devices[1]["ip"], - ["mac"] = breach_devices[1]["mac"], - ["subnet"] = breach_devices[1]["subnet"], - ["uid"] = breach_devices[1]["did"], - ["subnet_uid"] = breach_devices[1]["sid"], - ["type_id"] = 0 - } - end - - local observables_array = {} - for _, incident in ipairs(breach_devices) do - -- Hostname observable object - local hostname_val = incident["hostname"] or "" - if hostname_val ~= "" then - local hostname_obj = {} - hostname_obj["type_id"] = 1 - hostname_obj["type"] = "Hostname" - hostname_obj["name"] = "device.hostname" - hostname_obj["value"] = hostname_val - table.insert(observables_array, hostname_obj) - end - - -- IP Address observable object - local ip_val = incident["ip"] or "" - if ip_val ~= "" then - local ip_obj = {} - ip_obj["type_id"] = 2 - ip_obj["type"] = "IP Address" - ip_obj["name"] = "device.ip" - ip_obj["value"] = ip_val - table.insert(observables_array, ip_obj) - end - - -- MAC Address observable object - local mac_val = incident["mac"] or "" - if mac_val ~= "" then - local mac_obj = {} - mac_obj["type_id"] = 3 - mac_obj["type"] = "MAC Address" - mac_obj["name"] = "device.mac" - mac_obj["value"] = mac_val - table.insert(observables_array, mac_obj) - end - end - - if #observables_array > 0 then - fields["observables"] = observables_array - end - - return fields -end - -local function get_modelbreaches_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["original_time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or "") - }, - ["action"] = "Other", - ["action_id"] = 99, - ["activity_id"] = 1, - ["activity_name"] = "Create", - ["event.type"] = "Create", - ["category_name"] = "Findings", - ["category_uid"] = 2, - ["class_name"] = "Detection Finding", - ["class_uid"] = 2004, - ["type_name"] = "Detection Finding: Create", - ["type_uid"] = 200401, - ["severity_id"] = 99, - ["time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or "") - } - - fields["authorizations"] = { - { - ["policy"] = { - ["uid"] = event["pbid"] - } - } - } - - local model = event["model"] or {} - local model_then = model["then"] - -- In Lua, empty tables are true, so check if not nil. - -- If 'then' might be an empty table {}, you might need next(model_then) check, - -- but usually API responses omit keys or return nil if missing. - if model_then then - fields["resources"] = { - { - ["name"] = model_then["name"], - ["uid"] = model_then["uuid"], - ["version"] = model_then["version"], - ["data"] = model_then - } - } - end - - local model_now = model["now"] - -- Note: Original Python code had a bug: 'if model_then:' check for 'resources_result' - -- I assumed you wanted 'if model_now:' here based on context. - -- If you want strict translation of the bug, change 'model_now' to 'model_then' in the if condition. - if model_now then - fields["resources_result"] = { - { - ["name"] = model_now["name"], - ["uid"] = model_now["uuid"], - ["version"] = model_now["version"], - ["data"] = model_now - } - } - end - - local device = event["device"] or {} - local device_did = device["did"] - if device_did then - fields["device"] = { - ["uid"] = device_did, - ["type_id"] = 0 - } - end - - return fields -end - -local function get_status_logs_computed_fields(event) - local fields = { - ["dataSource"] = { - ["category"] = "security", - ["name"] = "Darktrace", - ["vendor"] = "Darktrace", - }, - ["metadata"] = { - ["product"] = { - ["name"] = "Darktrace", - ["vendor_name"] = "Darktrace", - }, - ["version"] = "1.1.0", - ["original_time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or "") - }, - ["device"] = { - ["type_id"] = 99, - ["type"] = event["type"], - ["hostname"] = event["hostname"], - ["ip"] = event["ipAddress"], - ["network_interfaces"] = { - { - ["ip"] = event["networkInterfacesAddress_eth0"], - ["type_id"] = 1, - ["type"] = "Wired" - } - } - }, - ["time"] = type(event["time"]) == "number" and event["time"] or convert_to_epoch_milliseconds(event["time"] or ""), - ["activity_id"] = 1, - ["activity_name"] = "Log", - ["event.type"] = "Log", - ["category_name"] = "Discovery", - ["category_uid"] = 5, - ["class_name"] = "Device Config State", - ["class_uid"] = 5002, - ["type_name"] = "Device Config State: Log", - ["type_uid"] = 500201, - ["severity_id"] = 99, - } - - -- Initialize device table - - local subnet_data = event["subnetData"] or {} - if #subnet_data >= 1 then - local sid = subnet_data[1]["sid"] - fields["device"]["subnet_uid"] = sid - end - - fields["observables"] = {} - - local hostname = event["hostname"] - if hostname and hostname ~= "" then - table.insert(fields["observables"], { - ["type_id"] = 1, - ["type"] = "Hostname", - ["name"] = "device.hostname", - ["value"] = hostname, - }) - end - - local ip_address = event["ipAddress"] - if ip_address and ip_address ~= "" then - table.insert(fields["observables"], { - ["type_id"] = 2, - ["type"] = "IP Address", - ["name"] = "device.ip", - ["value"] = ip_address, - }) - end - - return fields -end - - - - -local function getConstantFields(event_type, event) - if event_type == "aianalyst/groups" then - return get_group_logs_computed_fields(event) - elseif event_type == "aianalyst/incidentevents" then - return get_incidents_logs_computed_fields(event) - elseif event_type == "modelbreaches" then - return get_modelbreaches_logs_computed_fields(event) - elseif event_type == "status" then - return get_status_logs_computed_fields(event) - end -end - -local IGNORE_KEYS = { - _ob = true, - _darktrace_event_type = true, - _darktrace_query_time = true, - _darktrace_start_time = true, - _darktrace_end_time = true, - timestamp = true -} - - --- Utility functions -local function split(str, delimiter) - if not str or str == "" then - return {} - end - local result = {} - for token in string.gmatch(str, "([^" .. delimiter .. "]+)") do - table.insert(result, token) - end - return result -end - -local function getValueByPath(obj, path) - local current = obj - for _, part in ipairs(split(path, ".")) do - if type(current) ~= "table" then - return nil - end - local key = tonumber(part) or part - current = current[key] - end - return current -end - -local function setValueByPath(obj, path, value) - local parts = split(path, ".") - local current = obj - for i = 1, #parts - 1 do - local key = tonumber(parts[i]) or parts[i] - if current[key] == nil or type(current[key]) ~= "table" then - current[key] = {} - end - current = current[key] - end - local last = parts[#parts] - if value == nil then - current[last] = nil - else - current[last] = value - end -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - --- Helper: Encode Lua table to JSON string with field ordering -local function encodeJson(obj, fieldOrder, key) - if obj == nil then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray then - local items = {} - for i = 1, maxIndex do - table.insert(items, encodeJson(obj[i], fieldOrder, key)) - end - return "[" .. table.concat(items, ",") .. "]" - else - local items = {} - local fieldOrdering = fieldOrder[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrdering) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldOrder, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - -- Phase 2: remaining keys (not in fieldOrder) - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrdering) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, fieldOrder, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - return "{" .. table.concat(items, ",") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local GROUPS_FIELD_ORDER = { - message = { - "id", "active", "acknowledged", "pinned", "userTriggered", "externalTriggered", - "previousIds", "incidentEvents", "mitreTactics", "devices", "initialDevices", - "category", "groupScore", "start", "end", "edges" - }, - incidentEvents = { - "uuid", "start", "title", "triggerDid", "visible" - }, - edges = { - "isAction", "source", "target", "start", "incidentEvent", "description", "details" - }, - source = { - "nodeType", "value" - }, - target = { - "nodeType", "value" - } -} - -local INCIDENT_FIELD_ORDER = { - message = { - "summariser", "acknowledged", "pinned", "createdAt", "attackPhases", "mitreTactics", - "title", "id", "children", "category", "currentGroup", "groupCategory", "groupScore", - "groupPreviousGroups", "activityId", "groupingIds", "groupByActivity", "userTriggered", - "externalTriggered", "aiaScore", "summary", "periods", "breachDevices", "relatedBreaches", - "details" - }, - periods = { - "start", "end" - }, - breachDevices = { - "identifier", "hostname", "ip", "mac", "subnet", "did", "sid" - }, - relatedBreaches = { - "modelName", "pbid", "threatScore", "timestamp" - }, - details = { - "contents", "header" - }, - contents = { - "key", "type", "values" - }, - values = { - "identifier", "hostname", "ip", "mac", "subnet", "did", "sid" - } -} - -local MODEL_BREACH_FIELD_ORDER = { - message = { - "commentCount", "pbid", "time", "creationTime", "model", - "triggeredComponents", "score", "device" - }, - model = { - "then", "now" - }, - -- 'then' is a reserved keyword in Lua, so we use bracket notation - ["then"] = { - "name", "pid", "phid", "uuid", "logic", "throttle", "sharedEndpoints", - "actions", "tags", "interval", "delay", "sequenced", "active", "modified", - "activeTimes", "autoUpdatable", "autoUpdate", "autoSuppress", "description", - "behaviour", "defeats", "created", "edited", "message", "version", - "priority", "category", "compliance" - }, - now = { - "name", "pid", "phid", "uuid", "logic", "throttle", "sharedEndpoints", - "actions", "tags", "interval", "delay", "sequenced", "active", "modified", - "activeTimes", "autoUpdatable", "autoUpdate", "autoSuppress", "description", - "behaviour", "defeats", "created", "edited", "message", "version", - "priority", "category", "compliance" - }, - actions = { - "alert", "antigena", "breach", "model", "setPriority", "setTag", "setType" - }, - activeTimes = { - "devices", "tags", "type", "version" - }, - created = { "by" }, - edited = { "by" }, - triggeredComponents = { - "time", "cbid", "cid", "chid", "size", "threshold", "interval", - "logic", "version", "metric", "triggeredFilters" - }, - metric = { - "mlid", "name", "label" - }, - triggeredFilters = { - "cfid", "id", "filterType", "arguments", "comparatorType", "trigger" - }, - arguments = { "value" }, - trigger = { "value" }, - device = { "did" }, - - -- Logic structures (Shared by model logic and component logic) - logic = { - "data", "type", "version" - }, - -- Recursive data structure for component logic - data = { - "left", "operator", "right" - }, - left = { - "left", "operator", "right" - }, - right = { - "left", "operator", "right" - } -} - -local STATUS_FIELD_ORDER = { - message = { - "excessTraffic", "time", "installed", "mobileAppConfigured", "version", "ipAddress", - "modelsUpdated", "modelPackageVersion", "bundleVersion", "bundleDate", "bundleInstalledDate", - "hostname", "inoculation", "applianceOSCode", "license", "saasConnectorLicense", - "antigenaSaasLicense", "syslogTLSSHA1Fingerprint", "syslogTLSSHA256Fingerprint", - "antigenaNetworkEnabled", "antigenaNetworkLicense", "antigenaNetworkRunning", - "logIngestionReplicated", "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", - "logIngestionTypes", "logIngestionMatches", "licenseCounts", "type", "diskUtilization", - "uptime", "systemUptime", "load", "cpu", "memoryUsed", "dataQueue", "darkflowQueue", - "networkInterfacesState_eth0", "networkInterfacesAddress_eth0", - "networkInterfacesReceived_eth0", "networkInterfacesTransmitted_eth0", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", "bandwidth2WeekPeakString", - "processedBandwidthCurrent", "processedBandwidthCurrentString", "processedBandwidthAverage", - "processedBandwidthAverageString", "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "eventsPerMinuteCurrent", "probes", "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", "operatingSystems", - "newDevices4Weeks", "newDevices7Days", "newDevices24Hours", "newDevicesHour", - "activeDevices4Weeks", "activeDevices7Days", "activeDevices24Hours", "activeDevicesHour", - "deviceHostnames", "deviceMACAddresses", "deviceRecentIPChange", "models", "modelsBreached", - "modelsSuppressed", "devicesModeled", "recentUnidirectionalConnections", - "mostRecentCOTPTraffic", "mostRecentDCE_RPCTraffic", "mostRecentDHCPTraffic", - "mostRecentDNSTraffic", "mostRecentFTPTraffic", "mostRecentGSSAPITraffic", "mostRecentH2Traffic", - "mostRecentHTTPTraffic", "mostRecentHTTPSTraffic", "mostRecentKERBEROSTraffic", - "mostRecentLDAPTraffic", "mostRecentNETLOGONTraffic", "mostRecentNTLMTraffic", - "mostRecentNTPTraffic", "mostRecentRDPTraffic", "mostRecentSMBTraffic", "mostRecentSOCKSTraffic", - "mostRecentSSHTraffic", "mostRecentSSLTraffic", "ignoreAnalysisCredentials", - "internalIPRangeList", "internalIPRanges", "dnsServers", "internalDomains", - "internalAndExternalDomains", "proxyServers", "subnets", "subnetData" - }, - probes = {"1", "2", "3"}, - ["1"] = { - "id", "mappedId", "antigenaNetworkBlockedConnections", "configuredServer", "version", - "ipAddress", "bundleVersion", "bundleDate", "bundleInstalledDate", "metadata", "hostname", - "time", "installed", "kernel", "applianceOSCode", "syslogTLSSHA1Fingerprint", - "syslogTLSSHA256Fingerprint", "antigenaNetworkRunning", "logIngestionReplicated", - "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", "logIngestionDecryption", - "type", "diskUtilization", "uptime", "systemUptime", "load", "cpu", "memoryUsed", - "networkInterfacesState_ens5", "networkInterfacesAddress_ens5", - "networkInterfacesReceived_ens5", "networkInterfacesTransmitted_ens5", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", - "bandwidth2WeekPeakString", "processedBandwidthCurrent", "processedBandwidthCurrentString", - "processedBandwidthAverage", "processedBandwidthAverageString", - "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", - "label", "error", "lastContact" - }, - ["2"] = { - "id", "mappedId", "antigenaNetworkBlockedConnections", "configuredServer", "version", - "ipAddress", "bundleVersion", "bundleDate", "bundleInstalledDate", "metadata", "hostname", - "time", "installed", "kernel", "applianceOSCode", "syslogTLSSHA1Fingerprint", - "syslogTLSSHA256Fingerprint", "antigenaNetworkRunning", "logIngestionReplicated", - "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", "logIngestionDecryption", - "type", "diskUtilization", "uptime", "systemUptime", "load", "cpu", "memoryUsed", - "networkInterfacesState_ens5", "networkInterfacesAddress_ens5", - "networkInterfacesReceived_ens5", "networkInterfacesTransmitted_ens5", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", - "bandwidth2WeekPeakString", "processedBandwidthCurrent", "processedBandwidthCurrentString", - "processedBandwidthAverage", "processedBandwidthAverageString", - "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", - "label", "error", "lastContact" - }, - ["3"] = { - "id", "mappedId", "antigenaNetworkBlockedConnections", "configuredServer", "version", - "ipAddress", "bundleVersion", "bundleDate", "bundleInstalledDate", "metadata", "hostname", - "time", "installed", "kernel", "applianceOSCode", "syslogTLSSHA1Fingerprint", - "syslogTLSSHA256Fingerprint", "antigenaNetworkRunning", "logIngestionReplicated", - "logIngestionProcessed", "logIngestionTCP", "logIngestionUDP", "logIngestionDecryption", - "type", "diskUtilization", "uptime", "systemUptime", "load", "cpu", "memoryUsed", - "networkInterfacesState_ens5", "networkInterfacesAddress_ens5", - "networkInterfacesReceived_ens5", "networkInterfacesTransmitted_ens5", - "bandwidthCurrent", "bandwidthCurrentString", "bandwidthAverage", "bandwidthAverageString", - "bandwidth7DayPeak", "bandwidth7DayPeakString", "bandwidth2WeekPeak", - "bandwidth2WeekPeakString", "processedBandwidthCurrent", "processedBandwidthCurrentString", - "processedBandwidthAverage", "processedBandwidthAverageString", - "processedBandwidth7DayPeak", "processedBandwidth7DayPeakString", - "processedBandwidth2WeekPeak", "processedBandwidth2WeekPeakString", - "connectionsPerMinuteCurrent", "connectionsPerMinuteAverage", - "connectionsPerMinute7DayPeak", "connectionsPerMinute2WeekPeak", - "label", "error", "lastContact" - }, - - antigenaNetworkBlockedConnections = { - "attempted", "failed" - }, - - subnetData = { - "sid", "network", "devices", "clientDevices", "mostRecentTraffic", "mostRecentDHCP", "kerberosQuality" - }, - - eventsPerMinuteCurrent = { - "networkConnections", "logInputConnections", "cSensorConnections", "cSensorNotices", - "cSensorDeviceDetails", "cSensorModelEvents", "networkNotices", "networkDeviceDetails", - "networkModelEvents", "logInputNotices", "logInputDeviceDetails", "logInputModelEvents", - "saasNotices", "saasModelEvents" - }, - - licenseCounts = { - "saas", "licenseIPCount" - }, - - saas = { - "total" - }, - - -- Dynamic keys; listing those present in the log to ensure order if they appear - logIngestionTypes = { - "TestingDeviceObjects-connectionlogs", "TestingDeviceObjects2-connectionlogs", - "TestingDeviceObjects3-connectionlogs", "TestingDeviceObjects4-connectionlogs" - }, - - logIngestionMatches = { - "TestMatch", "TestSrcHostname", "TestingDeviceObjects-connectionlogs" - } -} - - -local function get_msg_field_ordering(event_type) - if event_type == "aianalyst/groups" then - return GROUPS_FIELD_ORDER - elseif event_type == "aianalyst/incidentevents" then - return INCIDENT_FIELD_ORDER - elseif event_type == "modelbreaches" then - return MODEL_BREACH_FIELD_ORDER - elseif event_type == "status" then - return STATUS_FIELD_ORDER - end -end - -local function cleanupEmptyNull(obj) - if type(obj) ~= "table" then - if type(obj) == "string" then - local lower = obj:lower() - if lower == "" or lower == "null" then - return nil - end - end - return obj - end - for key, value in pairs(obj) do - local cleaned = cleanupEmptyNull(value) - if cleaned == nil then - obj[key] = nil - else - obj[key] = cleaned - end - end - if next(obj) == nil then - return nil - end - return obj -end - -local function collectUnmapped(source, target) - for key, value in pairs(source or {}) do - if type(value) == "table" then - local nested = {} - collectUnmapped(value, nested) - if next(nested) then - target[key] = nested - end - else - target[key] = value - end - end -end - -local function getEventType(source) - if source["start"] then - return "aianalyst/groups" - elseif source["relatedBreaches"] then - return "aianalyst/incidentevents" - elseif source["time"] and source["triggeredComponents"] then - return "modelbreaches" - else - return "status" - end -end - -function processEvent(event) - if type(event) ~= "table" then - return nil - end - - local MAPPED_FIELDS = {} - - local source = deepCopy(event, IGNORE_KEYS) - - local event_type = getEventType(source) - - local result = {} - - for _, mapping in ipairs(COMMON_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - - if event_type == "aianalyst/groups" then - for _, mapping in ipairs(GROUPS_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - elseif event_type == "aianalyst/incidentevents" then - for _, mapping in ipairs(INCIDENTS_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - elseif event_type == "modelbreaches" then - for _, mapping in ipairs(MODEL_BREACHES_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - elseif event_type == "status" then - for _, mapping in ipairs(STATUS_LOGS_MAPPING) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - end - MAPPED_FIELDS[mapping.source] = true - end - end - - for key, value in pairs(getConstantFields(event_type, event)) do - result[key] = value - end - - -- Remove mapped fields from original event for unmapped collection - for key, _ in pairs(MAPPED_FIELDS) do - setValueByPath(event, key, nil) - end - - for key, _ in pairs(IGNORE_KEYS) do - setValueByPath(event, key, nil) - end - - - - local unmapped = {} - collectUnmapped(event, unmapped) - unmapped["event.type"] = event_type - result.unmapped = {} - if next(unmapped) then - -- Merge with existing unmapped - for k, v in pairs(unmapped) do - result.unmapped[k] = v - end - end - - result.message = encodeJson(source, get_msg_field_ordering(event_type), "message") - - if FEATURES.CLEANUP_EMPTY_NULL then - cleanupEmptyNull(result) - end - - return result -end \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/gcp_audit_logs.json b/pipelines/community/transform_ocsf/gcp_audit_logs/gcp_audit_logs.json deleted file mode 100644 index 5749ebc..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/gcp_audit_logs.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Gcp Audit Logs", - "grade": { - "letter": "F", - "score": 25, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Gcp Audit Logs. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "gcp", - "source_name": "gcp_audit_logs", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "gcp-audit-logs-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- GCP Audit Log OCSF 1.0.0 Schema Serializer\nlocal FEATURES = {\n FLATTEN_EVENT_TYPE = true,\n CLEANUP_EMPTY_NULL = true,\n}\n\nlocal BaseEvent = {\n CATEGORY_UID = 3,\n CATEGORY_NAME = \"Identity & Access Management\",\n TIMEZONE_OFFSET = 0,\n STATUS_ID = 0,\n STATUS = \"Unknown\"\n}\n\nlocal DataSourceEvent = {\n VENDOR = \"GCP\",\n NAME = \"GCP Audit\",\n CATEGORY = \"security\"\n}\n\nlocal MetaDataEvent = {\n LOG_PROVIDER = \"GCP Audit\",\n PRODUCT_LANG = \"en\",\n PRODUCT_NAME = \"GCP Audit\",\n PRODUCT_VENDOR_NAME = \"GCP\",\n VERSION = \"1.0.0\"\n}\n\nlocal Mapping = {\n SEVERITY = {\n [\"INFO\"] = {1, \"Informational\"},\n [\"CRITICAL\"] = {5, \"Critical\"}\n },\n ACTIVITY = {\n admin_activity = {99, \"AdminActivity\"},\n data_access = {99, \"DataAccess\"},\n system_event = {99, \"SystemEvent\"},\n policy_denied = {99, \"PolicyDenied\"}\n },\n TYPE = {\n admin_activity = {300499, \"Entity Management: AdminActivity\"},\n data_access = {300499, \"Entity Management: DataAccess\"},\n system_event = {300499, \"Entity Management: SystemEvent\"},\n policy_denied = {300599, \"User Access Management: PolicyDenied\"}\n },\n CLASS = {\n admin_activity = {3004, \"Entity Management\"},\n data_access = {3004, \"Entity Management\"},\n system_event = {3004, \"Entity Management\"},\n policy_denied = {3005, \"User Access Management\"}\n }\n}\n\nlocal MAPPED_FIELDS = {}\n\nlocal FIELD_ORDERS = {\n root = {\n \"activity_id\",\"activity_name\",\"category_uid\",\"category_name\",\"class_uid\",\"class_name\",\"severity_id\",\"type_uid\",\"type_name\",\"dataSource\",\"site.id\",\"metadata.product.name\",\"metadata.product.vendor_name\",\"metadata.version\",\"actor.user.email_addr\",\"device.ip\",\"api.service.name\",\"entity.name\",\"resource.name\",\"metadata.uid\",\"metadata.original_time\",\"metadata.logged_time\",\"metadata.log_name\",\"metadata.event_code\",\"api.operation\",\"cloud.provider\",\"event.type\",\"message\",\"status_code\",\"status\",\"status_detail\",\"time\"\n },\n metadata = {\"original_time\", \"logged_time\", \"log_name\", \"event_code\", \"uid\", \"product\", \"version\", \"currentLocations\", \"deviceState\", \"@type\", \"violationReason\", \"intermediateServices\", \"securityPolicyInfo\", \"resourceNames\", \"ingressViolations\", \"vpcServiceControlsUniqueId\"},\n product = {\"name\", \"vendor_name\"},\n dataSource = {\"category\", \"name\", \"vendor\"},\n site = {\"id\"},\n event = {\"type\"},\n message = {\"protoPayload\", \"insertId\", \"resource\", \"timestamp\", \"severity\", \"logName\", \"operation\", \"receiveTimestamp\"},\n protoPayload = {\"@type\", \"status\", \"authenticationInfo\", \"requestMetadata\", \"serviceName\", \"methodName\", \"authorizationInfo\", \"resourceName\", \"resourceLocation\", \"request\", \"metadata\"},\n status = {\"code\", \"message\", \"details\"},\n authenticationInfo = {\"principalEmail\"},\n requestMetadata = {\"callerIp\", \"callerSuppliedUserAgent\", \"requestAttributes\", \"destinationAttributes\"},\n requestAttributes = {\"time\", \"reason\", \"auth\"},\n authorizationInfo = {\"resource\", \"permission\", \"granted\", \"resourceAttributes\"},\n resource = {\"type\", \"labels\"},\n operation = {\"id\", \"producer\", \"first\"},\n labels = {\"method\", \"service\", \"project_id\", \"location\", \"bucket_name\", \"subnetwork_name\", \"subnetwork_id\"},\n request = {\"@type\"},\n preconditionFailure = {\"@type\", \"violations\"},\n details = {\"@type\", \"violations\"},\n violations = {\"description\", \"type\"},\n ingressViolations = {\"targetResource\", \"servicePerimeter\"},\n securityPolicyInfo = {\"organizationId\", \"servicePerimeterName\"},\n}\n\n\nlocal FIELD_MAPPINGS = {\n {source = \"activity_id\", target = \"activity_id\"},\n {source = \"activity_name\", target = \"activity_name\"},\n {source = \"category_uid\", target = \"category_uid\"},\n {source = \"category_name\", target = \"category_name\"},\n {source = \"class_uid\", target = \"class_uid\"},\n {source = \"class_name\", target = \"class_name\"},\n {source = \"severity_id\", target = \"severity_id\"},\n {source = \"type_uid\", target = \"type_uid\"},\n {source = \"status_code\", target = \"status_code\"},\n {source = \"status\", target = \"status\"},\n {source = \"status_detail\", target = \"status_detail\"},\n {source = \"dataSource\", target = \"dataSource\"},\n {source = \"site.id\", target = \"site.id\"},\n {source = \"metadata.product.name\", target = \"metadata.product.name\"},\n {source = \"metadata.product.vendor_name\", target = \"metadata.product.vendor_name\"},\n {source = \"metadata.version\", target = \"metadata.version\"},\n {source = \"metadata.uid\", target = \"metadata.uid\"},\n {source = \"metadata.original_time\", target = \"metadata.original_time\"},\n {source = \"metadata.logged_time\", target = \"metadata.logged_time\"},\n {source = \"metadata.log_name\", target = \"metadata.log_name\"},\n {source = \"metadata.event_code\", target = \"metadata.event_code\"},\n {source = \"actor.user.email_addr\", target = \"actor.user.email_addr\"},\n {source = \"device.ip\", target = \"device.ip\"},\n {source = \"api.service.name\", target = \"api.service.name\"},\n {source = \"api.operation\", target = \"api.operation\"},\n {source = \"entity.name\", target = \"entity.name\"},\n {source = \"resource.name\", target = \"resource.name\"},\n {source = \"cloud.provider\", target = \"cloud.provider\"},\n {source = \"time\", target = \"time\"},\n {source = \"message\", target = \"message\"}\n --{source = \"event.type\", target = \"event.type\"},\n\n}\n\nlocal IGNORE_KEYS = {\n _ob = true,\n message_id = true,\n}\n\nlocal function split(str, delimiter)\n if not str or str == \"\" then\n return {}\n end\n local result = {}\n for token in string.gmatch(str, \"([^\" .. delimiter .. \"]+)\") do\n table.insert(result, token)\n end\n return result\nend\n\nlocal function getValueByPath(obj, path)\n local current = obj\n for _, part in ipairs(split(path, \".\")) do\n\n if type(current) ~= \"table\" then\n return nil\n end\n\n local key = tonumber(part) or part\n\n current = current[key]\n end\n return current\nend\n\nlocal function setValueByPath(obj, path, value)\n local parts = split(path, \".\")\n local current = obj\n for i = 1, #parts - 1 do\n local key = tonumber(parts[i]) or parts[i]\n if current[key] == nil or type(current[key]) ~= \"table\" then\n current[key] = {}\n end\n current = current[key]\n end\n local last = parts[#parts]\n if value == nil then\n current[last] = nil\n else\n current[last] = value\n end\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n return value\n end\n\n local copy = {}\n for k, v in pairs(value) do\n if not (ignoreKeys and ignoreKeys[k]) then\n copy[k] = deepCopy(v, ignoreKeys)\n end\n end\n return copy\nend\n\nlocal function convertUtcToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\nlocal function encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" or obj == \"\" then\n return nil\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n\n if isArray then\n local items = {}\n for i = 1, maxIndex do\n local elementKey = key or tostring(i)\n local encoded = encodeJson(obj[i], elementKey)\n if encoded ~= nil then\n table.insert(items, encoded)\n end\n end\n if #items == 0 then\n return nil\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n local items = {}\n local fieldOrder = FIELD_ORDERS[key] or {}\n\n -- Phase 1: ordered keys\n for _, fieldName in ipairs(fieldOrder) do\n local v = obj[fieldName]\n if v ~= nil then\n local encoded = encodeJson(v, fieldName)\n if encoded ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\": ' .. encoded)\n end\n end\n end\n\n -- Phase 2: remaining keys\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then\n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local encoded = encodeJson(v, keyStr)\n if encoded ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. encoded)\n end\n end\n end\n\n if #items == 0 then\n return nil\n end\n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal function filterNullValues(obj)\n local filtered = {}\n for k, v in pairs(obj or {}) do\n if v ~= nil and v ~= \"\" and v ~= \"null\" then\n if type(v) == \"table\" then\n local nested = filterNullValues(v)\n if next(nested) then\n filtered[k] = nested\n end\n else\n filtered[k] = v\n end\n end\n end\n return filtered\nend\n\nlocal function isArrayTable(obj)\n if type(obj) ~= \"table\" then\n return false\n end\n local count = 0\n for k, _ in pairs(obj) do\n if type(k) ~= \"number\" then\n return false\n end\n count = count + 1\n end\n return count == #obj\nend\n\nlocal function applyFieldOrdering(obj, fieldOrder)\n if isArrayTable(obj) then\n local out = {}\n for i = 1, #obj do\n out[i] = obj[i]\n end\n return out\n end\n local ordered = {}\n for _, key in ipairs(fieldOrder or {}) do\n if obj[key] ~= nil then\n if type(obj[key]) == \"table\" then\n ordered[key] = applyFieldOrdering(obj[key], FIELD_ORDERS[key])\n else\n ordered[key] = obj[key]\n end\n end\n end\n for key, value in pairs(obj or {}) do\n local found = false\n for _, orderedKey in ipairs(fieldOrder or {}) do\n if orderedKey == key then\n found = true\n break\n end\n end\n if not found then\n if type(value) == \"table\" then\n ordered[key] = applyFieldOrdering(value, FIELD_ORDERS[key])\n else\n ordered[key] = value\n end\n end\n end\n return ordered\nend\n\nlocal function collectUnmapped(source, target, ignoreKeys)\n for key, value in pairs(source or {}) do\n if not (ignoreKeys and ignoreKeys[key]) then\n if type(value) == \"table\" then\n local nested = {}\n collectUnmapped(value, nested, ignoreKeys)\n if next(nested) then\n target[key] = nested\n end\n else\n target[key] = value\n end\n end\n end\nend\n\nlocal function getSeverityInfo(severity)\n local normalized = severity and severity:upper() or \"\"\n local info = Mapping.SEVERITY[normalized]\n if info then\n return info[1], info[2]\n end\n return 99, \"Unknown\" -- Default for unmapped severities (matches Python)\nend\n\nlocal function getEventType(logName)\n if not logName then\n return \"unknown\"\n end\n\n if string.find(logName, \"activity\") then\n return \"admin_activity\"\n elseif string.find(logName, \"data_access\") then\n return \"data_access\"\n elseif string.find(logName, \"system_event\") then\n return \"system_event\"\n elseif string.find(logName, \"policy\") then\n return \"policy_denied\"\n end\n\n return \"unknown\"\nend\n\nlocal function getActivityInfo(eventType)\n local info = Mapping.ACTIVITY[eventType]\n if info then\n return info[1], info[2]\n end\n return 99, \"Unknown\"\nend\n\nlocal function getTypeInfo(eventType)\n local info = Mapping.TYPE[eventType]\n if info then\n return info[1], info[2]\n end\n return 300499, \"Entity Management: Unknown\"\nend\n\nlocal function getClassInfo(eventType)\n local info = Mapping.CLASS[eventType]\n if info then\n return info[1], info[2]\n end\n return 3004, \"Entity Management\"\nend\n\nlocal function buildSource(event)\n local source = deepCopy(event, IGNORE_KEYS) or {}\n\n source.message = encodeJson(source, \"message\")\n\n -- Determine event type from logName\n local eventType = getEventType(source.logName)\n\n\n -- Get activity information\n local activity_id, activity_name = getActivityInfo(eventType)\n local type_uid, type_name = getTypeInfo(eventType)\n local class_uid, class_name = getClassInfo(eventType)\n\n -- Get severity information\n local severity_id, severity_label = getSeverityInfo(source.severity)\n\n -- Set base event properties\n source.activity_id = activity_id\n source.activity_name = activity_name\n source.category_uid = BaseEvent.CATEGORY_UID\n source.category_name = BaseEvent.CATEGORY_NAME\n source.class_uid = class_uid\n source.class_name = class_name\n \n source.severity_id = severity_id\n source.type_uid = type_uid\n --source.type_name = type_name\n\n -- Set data source\n source.dataSource = {\n vendor = DataSourceEvent.VENDOR,\n name = DataSourceEvent.NAME,\n category = DataSourceEvent.CATEGORY\n }\n\n -- Set metadata\n source.metadata = source.metadata or {}\n source.metadata.product = source.metadata.product or {}\n source.metadata.product.name = MetaDataEvent.PRODUCT_NAME\n source.metadata.product.vendor_name = MetaDataEvent.PRODUCT_VENDOR_NAME\n source.metadata.version = MetaDataEvent.VERSION\n\n -- Set site if available (siteId can be passed as parameter or extracted from log)\n source.site = source.site or {}\n source.site.id = source.site.id or \"\"\n\n -- Set event type\n source.event = {type = activity_name or \"\"}\n\n -- Map GCP audit log specific fields\n source.metadata.original_time = source.timestamp\n MAPPED_FIELDS[\"timestamp\"] = true\n source.metadata.logged_time = source.receiveTimestamp\n MAPPED_FIELDS[\"receiveTimestamp\"] = true\n source.metadata.log_name = source.logName\n MAPPED_FIELDS[\"logName\"] = true\n source.metadata.event_code = getValueByPath(source, \"operation.id\") or \"\"\n MAPPED_FIELDS[\"operation.id\"] = true\n source.metadata.uid = source.insertId\n MAPPED_FIELDS[\"insertId\"] = true\n -- Map actor and device information\n if source.protoPayload and source.protoPayload.authenticationInfo then\n source.actor = source.actor or {}\n source.actor.user = source.actor.user or {}\n source.actor.user.email_addr = source.protoPayload.authenticationInfo.principalEmail\n MAPPED_FIELDS[\"protoPayload.authenticationInfo.principalEmail\"] = true\n end\n\n if source.protoPayload and source.protoPayload.requestMetadata then\n source.device = source.device or {}\n source.device.ip = source.protoPayload.requestMetadata.callerIp\n MAPPED_FIELDS[\"protoPayload.requestMetadata.callerIp\"] = true\n end\n\n -- Map API information\n if source.protoPayload then\n source.api = source.api or {}\n source.api.service = source.api.service or {}\n source.api.service.name = source.protoPayload.serviceName\n MAPPED_FIELDS[\"protoPayload.serviceName\"] = true\n source.api.operation = getValueByPath(source, \"operation.producer\") or \"\"\n MAPPED_FIELDS[\"operation.producer\"] = true\n end\n\n -- Map resource information\n -- For policy denied events, use resource.name (not entity.name)\n -- For all other events, use entity.name (if resourceName exists)\n if source.protoPayload and source.protoPayload.resourceName then\n if eventType == \"policy_denied\" then\n source.resource = source.resource or {}\n source.resource.name = source.protoPayload.resourceName\n else\n source.entity = source.entity or {}\n source.entity.name = source.protoPayload.resourceName\n end\n MAPPED_FIELDS[\"protoPayload.resourceName\"] = true\n end\n\n -- Extract status fields from protoPayload.status (satisfies getpolicyDeniedOCSFMapping)\n if source.protoPayload and source.protoPayload.status then\n source.status_code = source.protoPayload.status.code\n source.status = source.protoPayload.status.message\n source.status_detail = source.protoPayload.status.details\n -- Mark status fields as mapped to avoid duplication\n MAPPED_FIELDS[\"protoPayload.status.code\"] = true\n MAPPED_FIELDS[\"protoPayload.status.message\"] = true\n MAPPED_FIELDS[\"protoPayload.status.details\"] = true\n end\n\n -- Set cloud provider\n source.cloud = source.cloud or {}\n source.cloud.provider = \"GCP\"\n\n\n -- Set time\n source.time = convertUtcToMilliseconds(source.timestamp)\n -- timestamp is already marked as mapped above\n\n -- Build message directly with ordering-aware encoder\n\n return source\nend\n\nlocal function cleanupEmptyNull(obj)\n\n if type(obj) ~= \"table\" then\n if type(obj) == \"string\" then\n local lower = obj:lower()\n if lower == \"\" or lower == \"null\" then\n return nil\n end\n end\n return obj\n end\n\n for key, value in pairs(obj) do\n local cleaned = cleanupEmptyNull(value)\n if cleaned == nil then\n obj[key] = nil\n else\n obj[key] = cleaned\n end\n end\n\n if next(obj) == nil then\n return nil\n end\n return obj\nend\n\nlocal function processEvent(event)\n if type(event) ~= \"table\" then\n return nil\n end\n\n -- reset mapped fields per event to avoid cross-event leakage\n MAPPED_FIELDS = {}\n\n local source = buildSource(event)\n\n local result = {}\n\n for _, mapping in ipairs(FIELD_MAPPINGS) do\n local value = getValueByPath(source, mapping.source)\n if value ~= nil then\n setValueByPath(result, mapping.target, deepCopy(value))\n MAPPED_FIELDS[mapping.source] = true\n end\n end\n\n -- Remove mapped fields from original event\n for key, _ in pairs(MAPPED_FIELDS) do\n setValueByPath(event, key, nil)\n end\n\n -- Collect remaining unmapped fields (fields not mapped to OCSF output)\n local unmapped = {}\n collectUnmapped(event, unmapped, IGNORE_KEYS)\n if next(unmapped) then\n result.unmapped = unmapped\n end\n\n if FEATURES.FLATTEN_EVENT_TYPE then\n if source and source.event then\n result['event.type'] = source.event.type\n end\n end\n\n if FEATURES.CLEANUP_EMPTY_NULL then\n cleanupEmptyNull(result)\n end\n\n return result\nend", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "F", - "harness_score": 25, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": false, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/metadata.yaml b/pipelines/community/transform_ocsf/gcp_audit_logs/metadata.yaml deleted file mode 100644 index 8ff5000..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/metadata.yaml +++ /dev/null @@ -1,47 +0,0 @@ -grade: - letter: F - score: 25 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Gcp Audit Logs. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: gcp - dataSource: Gcp Audit Logs - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "OAuth" - sample_record: "{\n \"kind\": \"admin#reports#activity\",\n \"id\": {\n \"time\": \"2026-04-20T03:32:52.967234+00:00\"\ - ,\n \"uniqueQualifier\": \"4878542320735016189\",\n \"applicationName\": \"admin\",\n \"\ - customerId\": \"C01NCC1701\"\n },\n \"etag\": \"\\\"d27c1b97272840a9b0d1738ada53d081\\\"\",\n \"\ - actor\": {\n \"email\": \"starfleet-admin@enterprise.starfleet.corp\",\n \"profileId\": \"328913592501844953\"\ - \n },\n \"ipAddress\": \"123.60.216.77\",\n \"events\": [\n {\n \"type\": \"USER_SETTINGS\"\ - ,\n \"name\": \"GRANT_ADMIN_ROLE\",\n \"parameters\": [\n {\n \"name\":\ - \ \"USER_EMAIL\",\n \"value\": \"jean.picard@starfleet.corp\"\n },\n {\n \ - \ \"name\": \"USER_NAME\",\n \"value\": \"Jean-Luc Picard\"\n },\n {\n\ - \ \"name\": \"SETTING_NAME\",\n \"value\": \"Calendar\"\n },\n {\n\ - \ \"name\": \"NEW_VALUE\",\n \"value\": \"ENABLED\"\n },\n {\n \ - \ \"name\": \"OLD_VALUE\",\n \"value\": \"DISABLED\"\n }\n ]\n }\n \ - \ ],\n \"ownerDomain\": \"starfleet.corp\"\n}" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: gcp, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: F - harness_score: 25 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/sample.json b/pipelines/community/transform_ocsf/gcp_audit_logs/sample.json deleted file mode 100644 index 8faddcf..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/sample.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "kind": "admin#reports#activity", - "id": { - "time": "2026-04-20T03:32:52.967234+00:00", - "uniqueQualifier": "4878542320735016189", - "applicationName": "admin", - "customerId": "C01NCC1701" - }, - "etag": "\"d27c1b97272840a9b0d1738ada53d081\"", - "actor": { - "email": "starfleet-admin@enterprise.starfleet.corp", - "profileId": "328913592501844953" - }, - "ipAddress": "123.60.216.77", - "events": [ - { - "type": "USER_SETTINGS", - "name": "GRANT_ADMIN_ROLE", - "parameters": [ - { - "name": "USER_EMAIL", - "value": "jean.picard@starfleet.corp" - }, - { - "name": "USER_NAME", - "value": "Jean-Luc Picard" - }, - { - "name": "SETTING_NAME", - "value": "Calendar" - }, - { - "name": "NEW_VALUE", - "value": "ENABLED" - }, - { - "name": "OLD_VALUE", - "value": "DISABLED" - } - ] - } - ], - "ownerDomain": "starfleet.corp" -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/gcp_audit_logs/serializer.lua b/pipelines/community/transform_ocsf/gcp_audit_logs/serializer.lua deleted file mode 100644 index 032bf88..0000000 --- a/pipelines/community/transform_ocsf/gcp_audit_logs/serializer.lua +++ /dev/null @@ -1,623 +0,0 @@ --- GCP Audit Log OCSF 1.0.0 Schema Serializer -local FEATURES = { - FLATTEN_EVENT_TYPE = true, - CLEANUP_EMPTY_NULL = true, -} - -local BaseEvent = { - CATEGORY_UID = 3, - CATEGORY_NAME = "Identity & Access Management", - TIMEZONE_OFFSET = 0, - STATUS_ID = 0, - STATUS = "Unknown" -} - -local DataSourceEvent = { - VENDOR = "GCP", - NAME = "GCP Audit", - CATEGORY = "security" -} - -local MetaDataEvent = { - LOG_PROVIDER = "GCP Audit", - PRODUCT_LANG = "en", - PRODUCT_NAME = "GCP Audit", - PRODUCT_VENDOR_NAME = "GCP", - VERSION = "1.0.0" -} - -local Mapping = { - SEVERITY = { - ["INFO"] = {1, "Informational"}, - ["CRITICAL"] = {5, "Critical"} - }, - ACTIVITY = { - admin_activity = {99, "AdminActivity"}, - data_access = {99, "DataAccess"}, - system_event = {99, "SystemEvent"}, - policy_denied = {99, "PolicyDenied"} - }, - TYPE = { - admin_activity = {300499, "Entity Management: AdminActivity"}, - data_access = {300499, "Entity Management: DataAccess"}, - system_event = {300499, "Entity Management: SystemEvent"}, - policy_denied = {300599, "User Access Management: PolicyDenied"} - }, - CLASS = { - admin_activity = {3004, "Entity Management"}, - data_access = {3004, "Entity Management"}, - system_event = {3004, "Entity Management"}, - policy_denied = {3005, "User Access Management"} - } -} - -local MAPPED_FIELDS = {} - -local FIELD_ORDERS = { - root = { - "activity_id","activity_name","category_uid","category_name","class_uid","class_name","severity_id","type_uid","type_name","dataSource","site.id","metadata.product.name","metadata.product.vendor_name","metadata.version","actor.user.email_addr","device.ip","api.service.name","entity.name","resource.name","metadata.uid","metadata.original_time","metadata.logged_time","metadata.log_name","metadata.event_code","api.operation","cloud.provider","event.type","message","status_code","status","status_detail","time" - }, - metadata = {"original_time", "logged_time", "log_name", "event_code", "uid", "product", "version", "currentLocations", "deviceState", "@type", "violationReason", "intermediateServices", "securityPolicyInfo", "resourceNames", "ingressViolations", "vpcServiceControlsUniqueId"}, - product = {"name", "vendor_name"}, - dataSource = {"category", "name", "vendor"}, - site = {"id"}, - event = {"type"}, - message = {"protoPayload", "insertId", "resource", "timestamp", "severity", "logName", "operation", "receiveTimestamp"}, - protoPayload = {"@type", "status", "authenticationInfo", "requestMetadata", "serviceName", "methodName", "authorizationInfo", "resourceName", "resourceLocation", "request", "metadata"}, - status = {"code", "message", "details"}, - authenticationInfo = {"principalEmail"}, - requestMetadata = {"callerIp", "callerSuppliedUserAgent", "requestAttributes", "destinationAttributes"}, - requestAttributes = {"time", "reason", "auth"}, - authorizationInfo = {"resource", "permission", "granted", "resourceAttributes"}, - resource = {"type", "labels"}, - operation = {"id", "producer", "first"}, - labels = {"method", "service", "project_id", "location", "bucket_name", "subnetwork_name", "subnetwork_id"}, - request = {"@type"}, - preconditionFailure = {"@type", "violations"}, - details = {"@type", "violations"}, - violations = {"description", "type"}, - ingressViolations = {"targetResource", "servicePerimeter"}, - securityPolicyInfo = {"organizationId", "servicePerimeterName"}, -} - - -local FIELD_MAPPINGS = { - {source = "activity_id", target = "activity_id"}, - {source = "activity_name", target = "activity_name"}, - {source = "category_uid", target = "category_uid"}, - {source = "category_name", target = "category_name"}, - {source = "class_uid", target = "class_uid"}, - {source = "class_name", target = "class_name"}, - {source = "severity_id", target = "severity_id"}, - {source = "type_uid", target = "type_uid"}, - {source = "status_code", target = "status_code"}, - {source = "status", target = "status"}, - {source = "status_detail", target = "status_detail"}, - {source = "dataSource", target = "dataSource"}, - {source = "site.id", target = "site.id"}, - {source = "metadata.product.name", target = "metadata.product.name"}, - {source = "metadata.product.vendor_name", target = "metadata.product.vendor_name"}, - {source = "metadata.version", target = "metadata.version"}, - {source = "metadata.uid", target = "metadata.uid"}, - {source = "metadata.original_time", target = "metadata.original_time"}, - {source = "metadata.logged_time", target = "metadata.logged_time"}, - {source = "metadata.log_name", target = "metadata.log_name"}, - {source = "metadata.event_code", target = "metadata.event_code"}, - {source = "actor.user.email_addr", target = "actor.user.email_addr"}, - {source = "device.ip", target = "device.ip"}, - {source = "api.service.name", target = "api.service.name"}, - {source = "api.operation", target = "api.operation"}, - {source = "entity.name", target = "entity.name"}, - {source = "resource.name", target = "resource.name"}, - {source = "cloud.provider", target = "cloud.provider"}, - {source = "time", target = "time"}, - {source = "message", target = "message"} - --{source = "event.type", target = "event.type"}, - -} - -local IGNORE_KEYS = { - _ob = true, - message_id = true, -} - -local function split(str, delimiter) - if not str or str == "" then - return {} - end - local result = {} - for token in string.gmatch(str, "([^" .. delimiter .. "]+)") do - table.insert(result, token) - end - return result -end - -local function getValueByPath(obj, path) - local current = obj - for _, part in ipairs(split(path, ".")) do - - if type(current) ~= "table" then - return nil - end - - local key = tonumber(part) or part - - current = current[key] - end - return current -end - -local function setValueByPath(obj, path, value) - local parts = split(path, ".") - local current = obj - for i = 1, #parts - 1 do - local key = tonumber(parts[i]) or parts[i] - if current[key] == nil or type(current[key]) ~= "table" then - current[key] = {} - end - current = current[key] - end - local last = parts[#parts] - if value == nil then - current[last] = nil - else - current[last] = value - end -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - -local function convertUtcToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - -local function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" or obj == "" then - return nil - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray then - local items = {} - for i = 1, maxIndex do - local elementKey = key or tostring(i) - local encoded = encodeJson(obj[i], elementKey) - if encoded ~= nil then - table.insert(items, encoded) - end - end - if #items == 0 then - return nil - end - return "[" .. table.concat(items, ", ") .. "]" - else - local items = {} - local fieldOrder = FIELD_ORDERS[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrder) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '": ' .. encoded) - end - end - end - - -- Phase 2: remaining keys - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. encoded) - end - end - end - - if #items == 0 then - return nil - end - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local function filterNullValues(obj) - local filtered = {} - for k, v in pairs(obj or {}) do - if v ~= nil and v ~= "" and v ~= "null" then - if type(v) == "table" then - local nested = filterNullValues(v) - if next(nested) then - filtered[k] = nested - end - else - filtered[k] = v - end - end - end - return filtered -end - -local function isArrayTable(obj) - if type(obj) ~= "table" then - return false - end - local count = 0 - for k, _ in pairs(obj) do - if type(k) ~= "number" then - return false - end - count = count + 1 - end - return count == #obj -end - -local function applyFieldOrdering(obj, fieldOrder) - if isArrayTable(obj) then - local out = {} - for i = 1, #obj do - out[i] = obj[i] - end - return out - end - local ordered = {} - for _, key in ipairs(fieldOrder or {}) do - if obj[key] ~= nil then - if type(obj[key]) == "table" then - ordered[key] = applyFieldOrdering(obj[key], FIELD_ORDERS[key]) - else - ordered[key] = obj[key] - end - end - end - for key, value in pairs(obj or {}) do - local found = false - for _, orderedKey in ipairs(fieldOrder or {}) do - if orderedKey == key then - found = true - break - end - end - if not found then - if type(value) == "table" then - ordered[key] = applyFieldOrdering(value, FIELD_ORDERS[key]) - else - ordered[key] = value - end - end - end - return ordered -end - -local function collectUnmapped(source, target, ignoreKeys) - for key, value in pairs(source or {}) do - if not (ignoreKeys and ignoreKeys[key]) then - if type(value) == "table" then - local nested = {} - collectUnmapped(value, nested, ignoreKeys) - if next(nested) then - target[key] = nested - end - else - target[key] = value - end - end - end -end - -local function getSeverityInfo(severity) - local normalized = severity and severity:upper() or "" - local info = Mapping.SEVERITY[normalized] - if info then - return info[1], info[2] - end - return 99, "Unknown" -- Default for unmapped severities (matches Python) -end - -local function getEventType(logName) - if not logName then - return "unknown" - end - - if string.find(logName, "activity") then - return "admin_activity" - elseif string.find(logName, "data_access") then - return "data_access" - elseif string.find(logName, "system_event") then - return "system_event" - elseif string.find(logName, "policy") then - return "policy_denied" - end - - return "unknown" -end - -local function getActivityInfo(eventType) - local info = Mapping.ACTIVITY[eventType] - if info then - return info[1], info[2] - end - return 99, "Unknown" -end - -local function getTypeInfo(eventType) - local info = Mapping.TYPE[eventType] - if info then - return info[1], info[2] - end - return 300499, "Entity Management: Unknown" -end - -local function getClassInfo(eventType) - local info = Mapping.CLASS[eventType] - if info then - return info[1], info[2] - end - return 3004, "Entity Management" -end - -local function buildSource(event) - local source = deepCopy(event, IGNORE_KEYS) or {} - - source.message = encodeJson(source, "message") - - -- Determine event type from logName - local eventType = getEventType(source.logName) - - - -- Get activity information - local activity_id, activity_name = getActivityInfo(eventType) - local type_uid, type_name = getTypeInfo(eventType) - local class_uid, class_name = getClassInfo(eventType) - - -- Get severity information - local severity_id, severity_label = getSeverityInfo(source.severity) - - -- Set base event properties - source.activity_id = activity_id - source.activity_name = activity_name - source.category_uid = BaseEvent.CATEGORY_UID - source.category_name = BaseEvent.CATEGORY_NAME - source.class_uid = class_uid - source.class_name = class_name - - source.severity_id = severity_id - source.type_uid = type_uid - --source.type_name = type_name - - -- Set data source - source.dataSource = { - vendor = DataSourceEvent.VENDOR, - name = DataSourceEvent.NAME, - category = DataSourceEvent.CATEGORY - } - - -- Set metadata - source.metadata = source.metadata or {} - source.metadata.product = source.metadata.product or {} - source.metadata.product.name = MetaDataEvent.PRODUCT_NAME - source.metadata.product.vendor_name = MetaDataEvent.PRODUCT_VENDOR_NAME - source.metadata.version = MetaDataEvent.VERSION - - -- Set site if available (siteId can be passed as parameter or extracted from log) - source.site = source.site or {} - source.site.id = source.site.id or "" - - -- Set event type - source.event = {type = activity_name or ""} - - -- Map GCP audit log specific fields - source.metadata.original_time = source.timestamp - MAPPED_FIELDS["timestamp"] = true - source.metadata.logged_time = source.receiveTimestamp - MAPPED_FIELDS["receiveTimestamp"] = true - source.metadata.log_name = source.logName - MAPPED_FIELDS["logName"] = true - source.metadata.event_code = getValueByPath(source, "operation.id") or "" - MAPPED_FIELDS["operation.id"] = true - source.metadata.uid = source.insertId - MAPPED_FIELDS["insertId"] = true - -- Map actor and device information - if source.protoPayload and source.protoPayload.authenticationInfo then - source.actor = source.actor or {} - source.actor.user = source.actor.user or {} - source.actor.user.email_addr = source.protoPayload.authenticationInfo.principalEmail - MAPPED_FIELDS["protoPayload.authenticationInfo.principalEmail"] = true - end - - if source.protoPayload and source.protoPayload.requestMetadata then - source.device = source.device or {} - source.device.ip = source.protoPayload.requestMetadata.callerIp - MAPPED_FIELDS["protoPayload.requestMetadata.callerIp"] = true - end - - -- Map API information - if source.protoPayload then - source.api = source.api or {} - source.api.service = source.api.service or {} - source.api.service.name = source.protoPayload.serviceName - MAPPED_FIELDS["protoPayload.serviceName"] = true - source.api.operation = getValueByPath(source, "operation.producer") or "" - MAPPED_FIELDS["operation.producer"] = true - end - - -- Map resource information - -- For policy denied events, use resource.name (not entity.name) - -- For all other events, use entity.name (if resourceName exists) - if source.protoPayload and source.protoPayload.resourceName then - if eventType == "policy_denied" then - source.resource = source.resource or {} - source.resource.name = source.protoPayload.resourceName - else - source.entity = source.entity or {} - source.entity.name = source.protoPayload.resourceName - end - MAPPED_FIELDS["protoPayload.resourceName"] = true - end - - -- Extract status fields from protoPayload.status (satisfies getpolicyDeniedOCSFMapping) - if source.protoPayload and source.protoPayload.status then - source.status_code = source.protoPayload.status.code - source.status = source.protoPayload.status.message - source.status_detail = source.protoPayload.status.details - -- Mark status fields as mapped to avoid duplication - MAPPED_FIELDS["protoPayload.status.code"] = true - MAPPED_FIELDS["protoPayload.status.message"] = true - MAPPED_FIELDS["protoPayload.status.details"] = true - end - - -- Set cloud provider - source.cloud = source.cloud or {} - source.cloud.provider = "GCP" - - - -- Set time - source.time = convertUtcToMilliseconds(source.timestamp) - -- timestamp is already marked as mapped above - - -- Build message directly with ordering-aware encoder - - return source -end - -local function cleanupEmptyNull(obj) - - if type(obj) ~= "table" then - if type(obj) == "string" then - local lower = obj:lower() - if lower == "" or lower == "null" then - return nil - end - end - return obj - end - - for key, value in pairs(obj) do - local cleaned = cleanupEmptyNull(value) - if cleaned == nil then - obj[key] = nil - else - obj[key] = cleaned - end - end - - if next(obj) == nil then - return nil - end - return obj -end - -local function processEvent(event) - if type(event) ~= "table" then - return nil - end - - -- reset mapped fields per event to avoid cross-event leakage - MAPPED_FIELDS = {} - - local source = buildSource(event) - - local result = {} - - for _, mapping in ipairs(FIELD_MAPPINGS) do - local value = getValueByPath(source, mapping.source) - if value ~= nil then - setValueByPath(result, mapping.target, deepCopy(value)) - MAPPED_FIELDS[mapping.source] = true - end - end - - -- Remove mapped fields from original event - for key, _ in pairs(MAPPED_FIELDS) do - setValueByPath(event, key, nil) - end - - -- Collect remaining unmapped fields (fields not mapped to OCSF output) - local unmapped = {} - collectUnmapped(event, unmapped, IGNORE_KEYS) - if next(unmapped) then - result.unmapped = unmapped - end - - if FEATURES.FLATTEN_EVENT_TYPE then - if source and source.event then - result['event.type'] = source.event.type - end - end - - if FEATURES.CLEANUP_EMPTY_NULL then - cleanupEmptyNull(result) - end - - return result -end \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/microsoft_365/metadata.yaml b/pipelines/community/transform_ocsf/microsoft_365/metadata.yaml deleted file mode 100644 index 642c3ce..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/metadata.yaml +++ /dev/null @@ -1,52 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Microsoft 365. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: microsoft - dataSource: Microsoft 365 - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "OAuth" - sample_record: "{\n \"id\": \"cb1628d2-7c07-4435-9f4e-87fd44024d05\",\n \"azureSubscriptionId\": \"\ - 6deb1e8b-896d-484a-a649-aeb6aebfeaf9\",\n \"azureTenantId\": \"8c7e1716-01d8-4aeb-ac89-c691e8957e6f\"\ - ,\n \"activityGroupName\": \"Suspicious email forwarding\",\n \"assignedTo\": \"unassigned\",\n\ - \ \"category\": \"MaliciousEmail\",\n \"closedDateTime\": \"2026-04-20T13:40:52.634172Z\",\n \"\ - cloudAppStates\": [\n {\n \"destinationServiceIp\": \"135.249.79.95\",\n \"destinationServiceName\"\ - : \"SharePoint Online\",\n \"riskScore\": \"66\"\n }\n ],\n \"comments\": [\n \"Alert\ - \ generated at 2026-04-20 03:33:52\",\n \"Escalated to security team\"\n ],\n \"confidence\"\ - : 97,\n \"createdDateTime\": \"2026-04-20T03:33:52.634172Z\",\n \"description\": \"Microsoft 365\ - \ security alert: Suspicious email forwarding. This alert indicates potential maliciousemail activity\ - \ detected in your environment.\",\n \"detectionIds\": [\n \"de01dafc-efd2-47a5-9d71-9ca354ae2bc5\"\ - \n ],\n \"eventDateTime\": \"2026-04-20T03:36:52.634172Z\",\n \"feedback\": \"falsePositive\",\n\ - \ \"incidentIds\": [],\n \"lastEventDateTime\": \"2026-04-20T03:54:52.634172Z\",\n \"lastModifiedDateTime\"\ - : \"2026-04-20T03:40:52.634172Z\",\n \"malwareStates\": [],\n \"networkConnections\": [\n {\n\ - \ \"applicationName\": \"chrome.exe\",\n \"destinationAddress\": \"199.227.89.52\",\n \ - \ \"destinationDomain\": \"graph.microsoft.com\",\n \"destinationPort\": \"8080\",\n \ - \ \"destinationUrl\": \"https://domain-62.com/api/data\",\n \"direction\": \"Outbound\",\n \ - \ \"domainRegisteredDateTime\": \"2025-07-18T03:40:52.634250Z\",\n " - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: microsoft, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/microsoft_365/microsoft_365.json b/pipelines/community/transform_ocsf/microsoft_365/microsoft_365.json deleted file mode 100644 index 42c323d..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/microsoft_365.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Microsoft 365", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Microsoft 365. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "microsoft", - "source_name": "microsoft_365", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "microsoft-365-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "-- Microsoft 365 (O365) OCSF 1.0.0 parser (ported from Python Parsers/microsoft)\n\n-- Helpers\n\nlocal function convertUtcToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n local year, month, day, hour, min, sec, frac =\n string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)\")\n if not year then\n return nil\n end\n local timeStruct = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n local localSeconds = os.time(timeStruct)\n local utcDate = os.date(\"!*t\", localSeconds)\n local adjustedSeconds\n if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and\n utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and\n utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then\n adjustedSeconds = localSeconds\n else\n local utcSeconds = os.time(utcDate)\n adjustedSeconds = localSeconds + (localSeconds - utcSeconds)\n end\n local milli = 0\n if frac and frac ~= \"\" then\n milli = tonumber((frac .. \"000\"):sub(1, 3))\n end\n return adjustedSeconds * 1000 + milli\nend\n\nlocal function split(str, delimiter)\n local result = {}\n local s = tostring(str or \"\")\n local escaped = delimiter:gsub(\"[%.%+%*%?%^%$%(%)%[%]%%]\", \"%%%1\")\n local pattern = \"([^\" .. escaped .. \"]+)\"\n for token in s:gmatch(pattern) do table.insert(result, token) end\n if #result == 0 and #s > 0 then table.insert(result, s) end\n return result\nend\n\nlocal function getByPath(obj, keys)\n local cur = obj\n for _, k in ipairs(keys) do\n if cur ~= nil and type(cur) == \"table\" then cur = cur[k] else return nil end\n end\n return cur\nend\n\nlocal function setByDottedPath(t, dotted, value)\n local keys = split(dotted, \".\")\n local cur = t\n for i=1,#keys-1 do\n local k = keys[i]\n if type(cur[k]) ~= \"table\" then cur[k] = {} end\n cur = cur[k]\n end\n cur[keys[#keys]] = value\nend\n\nlocal function flatten_table(tbl, prefix, out)\n out = out or {}\n prefix = prefix or \"\"\n\n for k, v in pairs(tbl) do\n local key\n\n -- ARRAY HANDLING (0-based index)\n if type(k) == \"number\" then\n key = string.format(\"%s[%d]\", prefix, k - 1)\n\n -- NORMAL OBJECT KEY\n else\n if prefix ~= \"\" then\n key = prefix .. \".\" .. k\n else\n key = k\n end\n end\n\n -- recurse\n if type(v) == \"table\" then\n flatten_table(v, key, out)\n else\n out[key] = v\n end\n end\n\n return out\nend\n\nlocal function apply_mapping(event, mapping)\n local out = {}\n\n -- normal mapping (existing behaviour)\n for src, dst in pairs(mapping) do\n local val = getByPath(event, split(src, \".\"))\n if val ~= nil then out[dst] = val end\n end\n\n -- NEW: auto-capture unmapped fields\n local flat_event = flatten_table(event)\n for src, val in pairs(flat_event) do\n if mapping[src] == nil then\n out[\"unmapped.\" .. src] = val\n end\n end\n\n return out\nend\n\nlocal function build_nested(flat)\n local nested = {}\n for k, v in pairs(flat) do setByDottedPath(nested, k, v) end\n return nested\nend\n\nlocal function cloneTable(src)\n local dst = {}\n for k,v in pairs(src or {}) do dst[k] = v end\n return dst\nend\n\n-- Simple email detection\nlocal function is_email(s)\n if type(s) ~= \"string\" then return false end\n return s:find(\"@\") ~= nil\nend\n\n-- Common + synthetic field helpers\nlocal function get_status_default_ocsf_mapping(status)\n if status == \"Success\" then return 1\n elseif status == \"Failure\" then return 2\n else return 99 end\nend\n\nlocal function get_device_os_type(platform)\n local mapping = { Win = 100, Android = 201, [\"iOS\"] = 301 }\n if type(platform) ~= \"string\" then return 99 end\n return mapping[platform] or 99\nend\n\nlocal function get_actor_type_details(user_type)\n local map = {\n [0] = { type = \"User\", id = 1 },\n [2] = { type = \"Admin\", id = 2 },\n [4] = { type = \"System\", id = 4 },\n }\n local rec = map[user_type]\n if rec then return rec.type, rec.id else return \"Other\", 99 end\nend\n\nlocal function get_state_id(event_type)\n if type(event_type) == \"string\" and event_type:lower() == \"dlprulematch\" then return 1 else return 0 end\nend\n\nlocal function extract_policy_details(log)\n local analytic_id, analytic_name = nil, nil\n local pd = log[\"PolicyDetails\"]\n if type(pd) == \"table\" and #pd > 0 and type(pd[1]) == \"table\" then\n if pd[1][\"PolicyId\"] then analytic_id = pd[1][\"PolicyId\"] end\n if pd[1][\"PolicyName\"] then analytic_name = pd[1][\"PolicyName\"] end\n end\n return analytic_id, analytic_name\nend\n\nlocal function get_device_property(log)\n local device_props = log[\"DeviceProperties\"]\n if type(device_props) == \"table\" then\n for _, dp in ipairs(device_props) do\n if type(dp) == \"table\" then\n if dp[\"Name\"] == \"OS\" then\n log[\"device_os_type\"] = dp[\"Value\"]\n log[\"device_os_type_id\"] = (dp[\"Value\"] == \"Linux\" and 200) or (dp[\"Value\"] == \"Windows\" and 100) or 99\n elseif dp[\"Name\"] == \"SessionId\" then\n log[\"actor_session_uid\"] = dp[\"Value\"]\n elseif dp[\"Name\"] == \"IsCompliantAndManaged\" then\n log[\"device_is_compliant\"] = dp[\"Value\"]\n log[\"device_is_managed\"] = dp[\"Value\"]\n end\n end\n end\n end\n return log\nend\n\nlocal function get_resource_data(log)\n local out = {}\n local mprops = log[\"ModifiedProperties\"]\n if type(mprops) == \"table\" then\n for _, obj in ipairs(mprops) do\n if type(obj) == \"table\" and obj[\"Name\"] then\n out[obj[\"Name\"]] = { NewValue = obj[\"NewValue\"], OldValue = obj[\"OldValue\"] }\n end\n end\n end\n return out\nend\n\nlocal function get_mgmt_observables(log)\n local obs = {}\n local sp = log[\"SharePointMetaData\"]\n if type(sp) == \"table\" then\n if sp[\"FileName\"] then table.insert(obs, { type_id = 7, type = \"FileName\", name = \"process.file.name\", value = sp[\"FileName\"] }) end\n if sp[\"From\"] then table.insert(obs, { type_id = 4, type = \"User Name\", name = \"unmapped.sharePointMetaData.from\", value = sp[\"From\"] }) end\n end\n return obs\nend\n\n-- Synthetic for Graph API\nlocal function set_graph_synthetic_fields(log, site_id)\n log[\"category_uid\"] = 2\n log[\"class_uid\"] = 2001\n log[\"class_name\"] = \"Security Finding\"\n log[\"type_uid\"] = 200199\n log[\"type_name\"] = \"Security Finding: Other\"\n log[\"OCSF_version\"] = \"1.0.0\"\n log[\"category_name\"] = \"Findings\"\n log[\"activity_id\"] = 99\n log[\"dataSource\"] = { name = \"Microsoft O365\", category = \"security\", vendor = \"Microsoft\" }\n log[\"event\"] = { type = log[\"status\"] }\n log[\"cloud\"] = { provider = \"Microsoft Azure\" }\n if site_id then log[\"site\"] = { id = site_id } end\n -- observables\n local observables = {}\n if log[\"azureTenantId\"] then table.insert(observables, { type_id = 10, type = \"Resource UID\", name = \"unmapped.azureTenantId\", value = log[\"azureTenantId\"] }) end\n log[\"observables\"] = observables\n if type(log[\"sourceMaterials\"]) == \"table\" then log[\"sourceMaterials\"] = log[\"sourceMaterials\"][1] end\n return log\nend\n\n-- Severity for Mgmt\nlocal function get_severity_id(event_type)\n local m = {\n dlprulematch = 1, dlpruleundo = 0, [\"update user\"] = 0, userloggedin = 0, [\"add group\"] = 0,\n [\"add member to group\"] = 0, [\"delete group\"] = 0, [\"update group\"] = 0, [\"remove member from group\"] = 0,\n [\"add user\"] = 0, [\"reset user password\"] = 0, [\"delete user\"] = 0, addedtogroup = 0, signinevent = 0,\n }\n return m[(event_type or \"\"):lower()] or 0\nend\n\nlocal function get_mgmt_log_mapping_fields(event_type)\n local map = {\n [\"dlprulematch\"] = {\n activity = { name = \"Create\", id = 99 }, class = { name = \"Security Finding\", id = 2001 }, category = { name = \"Findings\", id = 2 },\n },\n [\"dlpruleundo\"] = {\n activity = { name = \"DLPRuleUndo\", id = 99 }, class = { name = \"Security Finding\", id = 2001 }, category = { name = \"Findings\", id = 2 },\n },\n [\"update user\"] = {\n activity = { name = \"Update user\", id = 99 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"userloggedin\"] = {\n activity = { name = \"Logon\", id = 1 }, class = { name = \"Authentication\", id = 3002 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"add group\"] = {\n activity = { name = \"Add group\", id = 99 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"add member to group\"] = {\n activity = { name = \"Add User\", id = 3 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"delete group\"] = {\n activity = { name = \"Delete group\", id = 99 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"update group\"] = {\n activity = { name = \"Update group.\", id = 99 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"remove member from group\"] = {\n activity = { name = \"Remove User\", id = 4 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"add user\"] = {\n activity = { name = \"Create\", id = 1 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"reset user password\"] = {\n activity = { name = \"Password Reset\", id = 4 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"delete user\"] = {\n activity = { name = \"Delete\", id = 6 }, class = { name = \"Account Change\", id = 3001 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"addedtogroup\"] = {\n activity = { name = \"Add User\", id = 3 }, class = { name = \"Group Management\", id = 3006 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n [\"signinevent\"] = {\n activity = { name = \"Logon\", id = 1 }, class = { name = \"Authentication\", id = 3002 }, category = { name = \"Identity & Access Management\", id = 3 },\n },\n }\n local m = map[(event_type or \"\"):lower()] or { activity = { name = \"Other\", id = 99 }, class = { name = \"Base Event\", id = 0}, category = { name = \"Uncategorized\", id = 0 } }\n local class_name, class_id = m.class.name, m.class.id\n local activity_name, activity_id = m.activity.name, m.activity.id\n local type_name, type_id = string.format(\"%s: %s\", class_name, activity_name), (class_id * 100) + activity_id\n local category_name, category_id = m.category.name, m.category.id\n return activity_name, activity_id, class_name, class_id, type_name, type_id, category_name, category_id\nend\n\n-- Mapping tables from python\nlocal function get_default_graph_ocsf_mapping()\n return {\n [\"activityGroupName\"] = \"resources.group_name\",\n [\"category\"] = \"finding.types\",\n [\"closedDateTime\"] = \"end_time_dt\",\n [\"confidence\"] = \"confidence\",\n [\"createdDateTime\"] = \"metadata.original_time\",\n [\"eventDateTime\"] = \"start_time_dt\",\n [\"id\"] = \"metadata.uid\",\n [\"lastModifiedDateTime\"] = \"finding.modified_time\",\n [\"sourceMaterials\"] = \"finding.src_url\",\n [\"status\"] = \"activity_name\",\n [\"vendorInformation.provider\"] = \"metadata.product.name\",\n [\"vendorInformation.providerVersion\"] = \"metadata.product.version\",\n [\"vendorInformation.vendor\"] = \"metadata.product.vendor_name\",\n [\"lastEventDateTime\"] = \"end_time\",\n [\"title\"] = \"finding.title\",\n [\"category_uid\"] = \"category_uid\",\n [\"class_uid\"] = \"class_uid\",\n [\"type_uid\"] = \"type_uid\",\n [\"category_name\"] = \"category_name\",\n [\"class_name\"] = \"class_name\",\n [\"type_name\"] = \"type_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"cloud.provider\"] = \"cloud.provider\",\n [\"OCSF_version\"] = \"metadata.version\",\n [\"observables\"] = \"observables\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"event.type\"] = \"event.type\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n }\nend\n\nlocal function generic_mgmt_mapping()\n return {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n }\nend\n \nlocal function get_mgmt_default_mapping(event_type)\n -- ported from python OCSFMapping.get_mgmt_default_mapping\n local default = {\n [\"dlprulematch\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"finding.title\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"IncidentId\"] = \"finding.uid\",\n [\"SharePointMetaData.FileID\"] = \"process.file.uid\",\n [\"SharePointMetaData.FileName\"] = \"process.file.name\",\n [\"SharePointMetaData.FileOwner\"] = \"process.file.owner.uid\",\n [\"SharePointMetaData.FilePathUrl\"] = \"process.file.path\",\n [\"SharePointMetaData.FileSize\"] = \"process.file.size\",\n [\"SharePointMetaData.From\"] = \"resources.data.from\",\n [\"SharePointMetaData.IsViewableByExternalUsers\"] = \"resources.data.is_viewable_by_external_users\",\n [\"SharePointMetaData.IsVisibleOnlyToOdbOwner\"] = \"resources.data.is_visible_only_to_Odb_owner\",\n [\"SharePointMetaData.ItemCreationTime\"] = \"finding.created_time\",\n [\"SharePointMetaData.ItemLastModifiedTime\"] = \"finding.modified_time\",\n [\"SharePointMetaData.ItemLastSharedTime\"] = \"resources.data.item_last_shared_time\",\n [\"SharePointMetaData.SensitivityLabelIds\"] = \"resources.data.sensitivity_label_ids\",\n [\"SharePointMetaData.SharedBy\"] = \"resources.data.shared_by\",\n [\"SharePointMetaData.SiteAdmin\"] = \"resources.data.site_admin\",\n [\"SharePointMetaData.SiteCollectionGuid\"] = \"resources.data.site_collection_guid\",\n [\"SharePointMetaData.SiteCollectionUrl\"] = \"resources.data.site_collection_url\",\n [\"SharePointMetaData.UniqueID\"] = \"resources.data.unique_id\",\n [\"state_id\"] = \"state_id\",\n [\"status_id\"] = \"status_id\",\n [\"analytic.uid\"] = \"analytic.uid\",\n [\"analytic.name\"] = \"analytic.name\",\n [\"analytic_type_id\"] = \"analytic.type_id\",\n [\"process_file_owner_type_id\"] = \"process.file.owner.type_id\",\n },\n [\"dlpruleundo\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"finding.title\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"IncidentId\"] = \"finding.uid\",\n [\"SharePointMetaData.FileID\"] = \"process.file.uid\",\n [\"SharePointMetaData.FileName\"] = \"process.file.name\",\n [\"SharePointMetaData.FileOwner\"] = \"process.file.owner.uid\",\n [\"SharePointMetaData.FilePathUrl\"] = \"process.file.path\",\n [\"SharePointMetaData.FileSize\"] = \"process.file.size\",\n [\"SharePointMetaData.From\"] = \"resources.data.from\",\n [\"SharePointMetaData.IsViewableByExternalUsers\"] = \"resources.data.is_viewable_by_external_users\",\n [\"SharePointMetaData.IsVisibleOnlyToOdbOwner\"] = \"resources.data.is_visible_only_to_Odb_owner\",\n [\"SharePointMetaData.ItemCreationTime\"] = \"finding.created_time\",\n [\"SharePointMetaData.ItemLastModifiedTime\"] = \"finding.modified_time\",\n [\"SharePointMetaData.ItemLastSharedTime\"] = \"resources.data.item_last_shared_time\",\n [\"SharePointMetaData.SensitivityLabelIds\"] = \"resources.data.sensitivity_label_ids\",\n [\"SharePointMetaData.SharedBy\"] = \"resources.data.shared_by\",\n [\"SharePointMetaData.SiteAdmin\"] = \"resources.data.site_admin\",\n [\"SharePointMetaData.SiteCollectionGuid\"] = \"resources.data.site_collection_guid\",\n [\"SharePointMetaData.SiteCollectionUrl\"] = \"resources.data.site_collection_url\",\n [\"SharePointMetaData.UniqueID\"] = \"resources.data.unique_id\",\n [\"analytic.uid\"] = \"analytic.uid\",\n [\"analytic.name\"] = \"analytic.name\",\n [\"analytic_type_id\"] = \"analytic.type_id\",\n [\"process_file_owner_type_id\"] = \"process.file.owner.type_id\",\n [\"state_id\"] = \"state_id\",\n },\n [\"update user\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"activity_name\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"userloggedin\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ClientIP\"] = \"device.ip\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"ActorIpAddress\"] = \"src_endpoint.ip\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"ApplicationId\"] = \"dst_endpoint.uid\",\n [\"ErrorNumber\"] = \"api.response.code\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"device_os_type\"] = \"device.os.type\",\n [\"device_os_type_id\"] = \"device.os.type_id\",\n [\"actor_session_uid\"] = \"actor.session.uid\",\n [\"device_is_compliant\"] = \"device.is_compliant\",\n [\"device_is_managed\"] = \"device.is_managed\",\n },\n [\"add group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"activity_name\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"add member to group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"delete group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"update group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"Operation\"] = \"activity_name\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"remove member from group\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"resource.data\"] = \"resource.data\",\n },\n [\"add user\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"reset user password\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"delete user\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.UserKey\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ResultStatus\"] = \"status_detail\",\n [\"ActorContextId\"] = \"actor.user.org.uid\",\n [\"InterSystemsId\"] = \"metadata.correlation_uid\",\n [\"TargetContextId\"] = \"user.org.uid\",\n [\"status_id\"] = \"status_id\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n },\n [\"addedtogroup\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ClientIP\"] = \"device.ip\",\n [\"CorrelationId\"] = \"metadata.correlation_uid\",\n [\"EventSource\"] = \"resource.name\",\n [\"EventData\"] = \"status_detail\",\n [\"TargetUserOrGroupType\"] = \"user.groups.type\",\n [\"TargetUserOrGroupName\"] = \"user.groups.name\",\n [\"AppAccessContext.CorrelationId\"] = \"actor.idp.uid\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n },\n [\"signinevent\"] = {\n [\"CreationTime\"] = \"metadata.original_time\",\n [\"Id\"] = \"metadata.uid\",\n [\"OrganizationId\"] = \"cloud.org.uid\",\n [\"UserKey\"] = \"actor.user.credential_uid\",\n [\"Version\"] = \"metadata.product.version\",\n [\"Workload\"] = \"metadata.product.name\",\n [\"ClientIP\"] = \"src_endpoint.ip\",\n [\"CorrelationId\"] = \"metadata.correlation_uid\",\n [\"EventSource\"] = \"service.name\",\n [\"IsManagedDevice\"] = \"device.is_managed\",\n [\"Platform\"] = \"device.os.type\",\n [\"UserAgent\"] = \"http_request.user_agent\",\n [\"DeviceDisplayName\"] = \"device.ip\",\n [\"AppAccessContext.AADSessionId\"] = \"actor.session.uid\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"device_os_type_id\"] = \"device.os.type_id\",\n },\n }\n local et = (event_type or \"\"):lower()\n return default[et] or generic_mgmt_mapping()\nend\n\nlocal function common_mapping()\n return {\n [\"category_name\"] = \"category_name\",\n [\"category_uid\"] = \"category_uid\",\n [\"class_uid\"] = \"class_uid\",\n [\"severity_id\"] = \"severity_id\",\n [\"activity_name\"] = \"activity_name\",\n [\"activity_id\"] = \"activity_id\",\n [\"type_uid\"] = \"type_uid\",\n [\"product_vendor_name\"] = \"metadata.product.vendor_name\",\n [\"product_name\"] = \"metadata.product.name\",\n [\"OCSF_version\"] = \"metadata.version\",\n [\"observables\"] = \"observables\",\n [\"dataSource.category\"] = \"dataSource.category\",\n [\"site.id\"] = \"site.id\",\n [\"event.type\"] = \"event.type\",\n [\"dataSource.name\"] = \"dataSource.name\",\n [\"dataSource.vendor\"] = \"dataSource.vendor\",\n [\"message\"] = \"message\",\n [\"class_name\"] = \"class_name\",\n [\"type_name\"] = \"type_name\",\n [\"actor_user_email_addr\"] = \"actor.user.email_addr\",\n [\"actor_user_uid\"] = \"actor.user.uid\",\n [\"actor_user_type_id\"] = \"actor.user.type_id\",\n [\"actor_user_type\"] = \"actor.user.type\",\n [\"user_email_addr\"] = \"user.email_addr\",\n [\"user_uid\"] = \"user.uid\",\n [\"user_type_id\"] = \"user.type_id\",\n [\"user_type\"] = \"user.type\",\n [\"status_id\"] = \"status_id\",\n }\nend\n\n-- Mgmt synthetic enrichment\nlocal function set_mgmt_synthetic_fields(log, site_id)\n log[\"product_vendor_name\"] = \"Microsoft\"\n log[\"OCSF_version\"] = \"1.0.0\"\n local event_type = tostring(log[\"Operation\"] or \"Other\")\n local event_type_dup = event_type:gsub(\"%.\", \"\")\n\n log[\"severity_id\"] = get_severity_id(event_type_dup)\n local activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid = get_mgmt_log_mapping_fields(event_type_dup)\n log[\"activity_name\"], log[\"activity_id\"], log[\"class_name\"], log[\"class_uid\"], log[\"type_name\"], log[\"type_uid\"], log[\"category_name\"], log[\"category_uid\"] = activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid\n\n log[\"status_id\"] = get_status_default_ocsf_mapping(log[\"ResultStatus\"])\n\n if site_id then log[\"site\"] = { id = site_id } end\n\n if log[\"activity_id\"] == 99 then\n log[\"event\"] = { type = event_type }\n log[\"activity_name\"] = event_type\n else\n log[\"event\"] = { type = log[\"activity_name\"] }\n end\n\n log[\"dataSource\"] = { name = \"Microsoft O365\", category = \"security\", vendor = \"Microsoft\" }\n\n if event_type_dup:lower() == \"dlprulematch\" or event_type_dup:lower() == \"dlpruleundo\" then\n log[\"analytic_type_id\"] = 1\n log[\"process_file_owner_type_id\"] = 99\n log[\"observables\"] = get_mgmt_observables(log)\n log[\"state_id\"] = get_state_id(event_type_dup)\n local aid, aname = extract_policy_details(log)\n log[\"analytic\"] = { uid = aid, name = aname }\n else\n -- set actor values\n if (event_type_dup:lower() == \"userloggedin\") then\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = \"User\", 1\n end\n local userId = log[\"UserId\"]\n if is_email(userId) then\n log[\"actor_user_email_addr\"] = userId\n elseif type(userId) == \"string\" and userId ~= \"\" then\n log[\"actor_user_uid\"] = userId\n end\n local actors = log[\"Actor\"]\n if type(actors) == \"table\" and #actors > 0 then\n for _, a in ipairs(actors) do\n if type(a) == \"table\" then\n local id = a[\"ID\"]\n if is_email(id) then\n log[\"actor_user_email_addr\"] = id\n local t, tid = get_actor_type_details(a[\"Type\"])\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = t, tid\n break\n elseif type(id) == \"string\" and id:match(\"-\") then\n log[\"actor_user_uid\"] = id\n local t, tid = get_actor_type_details(a[\"Type\"])\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = t, tid\n break\n end\n end\n end\n else\n local t, tid = get_actor_type_details(log[\"UserType\"])\n log[\"actor_user_type\"], log[\"actor_user_type_id\"] = t, tid\n end\n\n -- set user target data\n local target = log[\"Target\"]\n if type(target) == \"table\" and #target > 0 and type(target[1]) == \"table\" then\n local tid = target[1][\"ID\"]\n if is_email(tid) then log[\"user_email_addr\"] = tid else log[\"user_uid\"] = tid end\n local t, tidv = get_actor_type_details(target[1][\"Type\"])\n log[\"user_type\"], log[\"user_type_id\"] = t, tidv\n end\n end\n\n if log[\"UserKey\"] == \"Not Available\" or log[\"UserKey\"] == \"NA\" then log[\"UserKey\"] = \"\" end\n\n local etl = event_type_dup:lower()\n if etl == \"add group\" or etl == \"add member to group\" or etl == \"delete group\" or etl == \"update group\" or etl == \"remove member from group\" then\n log[\"resource\"] = { data = get_resource_data(log) }\n end\n\n if etl == \"signinevent\" then\n log[\"device_os_type_id\"] = get_device_os_type(log[\"Platform\"])\n end\n\n if etl == \"userloggedin\" then\n log = get_device_property(log)\n end\n\n return log\nend\n\n-- Parsers\nlocal function default_graph_api_parser(alertLog)\n return build_nested(apply_mapping(alertLog, get_default_graph_ocsf_mapping()))\nend\n\nlocal function default_mgmt_api_parser(log)\n local event_type = tostring(log[\"Operation\"] or \"Other\"):gsub(\"%.\", \"\")\n local mapping = get_mgmt_default_mapping(event_type)\n -- merge with common mapping\n for src, dst in pairs(common_mapping()) do mapping[src] = dst end\n local flat = apply_mapping(log, mapping)\n return build_nested(flat)\nend\n\n-- API detection: Graph has createdDateTime, vendorInformation; Mgmt has CreationTime/Operation\nlocal function detect_api(event)\n if event[\"createdDateTime\"] or getByPath(event, {\"vendorInformation\"}) then return \"Graph\" end\n return \"Mgmt\"\nend\n\n-- Helper: Encode Lua table to JSON string with field ordering\nlocal function encodeJson(obj, fieldOrder, key)\n if obj == nil then\n\t return \"null\"\n elseif type(obj) == \"boolean\" then\n\t return tostring(obj)\n elseif type(obj) == \"number\" then\n\t return tostring(obj)\n elseif type(obj) == \"string\" then\n\t return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n\t local isArray = true\n\t local maxIndex = 0\n\t for k, v in pairs(obj) do\n\t\t if type(k) ~= \"number\" then\n\t\t\t isArray = false\n\t\t\t break\n\t\t end\n\t\t maxIndex = math.max(maxIndex, k)\n\t end\n\t if isArray then\n\t\t local items = {}\n\t\t for i = 1, maxIndex do\n\t\t\t table.insert(items, encodeJson(obj[i], fieldOrder, key))\n\t\t end\n\t\t return \"[\" .. table.concat(items, \",\") .. \"]\"\n\t else\n\t\t local items = {}\n\t\t local fieldOrdering = fieldOrder[key] or {}\n\t\t \n\t\t -- Phase 1: ordered keys\n\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t local v = obj[fieldName]\n\t\t\t if v ~= nil then\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, fieldName)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t -- Phase 2: remaining keys (not in fieldOrder)\n\t\t for k, v in pairs(obj) do\n\t\t\t local found = false\n\t\t\t for _, fieldName in ipairs(fieldOrdering) do\n\t\t\t\t if k == fieldName then\n\t\t\t\t\t found = true\n\t\t\t\t\t break\n\t\t\t\t end\n\t\t\t end\n\t\t\t if not found then\n\t\t\t\t local keyStr = type(k) == \"string\" and k or tostring(k)\n\t\t\t\t local encoded = encodeJson(v, fieldOrder, keyStr)\n\t\t\t\t if encoded ~= nil then\n\t\t\t\t\t table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\":' .. encoded)\n\t\t\t\t end\n\t\t\t end\n\t\t end\n\t\t \n\t\t return \"{\" .. table.concat(items, \",\") .. \"}\"\n\t end\n else\n\t return '\"' .. tostring(obj) .. '\"'\n end\nend\n\nlocal function deepCopy(value, ignoreKeys)\n if type(value) ~= \"table\" then\n\t return value\n end\n local copy = {}\n for k, v in pairs(value) do\n\t if not (ignoreKeys and ignoreKeys[k]) then\n\t\t copy[k] = deepCopy(v, ignoreKeys)\n\t end\n end\n return copy\nend\n\nlocal GRAPH_FIELD_ORDER = {\n message = {\n \"id\", \"azureTenantId\", \"azureSubscriptionId\", \"riskScore\", \"tags\", \"activityGroupName\", \"assignedTo\",\n \"category\", \"closedDateTime\", \"comments\", \"confidence\", \"createdDateTime\", \"description\", \"detectionIds\",\n \"eventDateTime\", \"feedback\", \"incidentIds\", \"lastEventDateTime\", \"lastModifiedDateTime\", \"recommendedActions\",\n \"severity\", \"sourceMaterials\", \"status\", \"title\", \"vendorInformation\", \"alertDetections\",\n \"cloudAppStates\", \"fileStates\", \"hostStates\", \"historyStates\", \"investigationSecurityStates\",\n \"malwareStates\", \"messageSecurityStates\", \"networkConnections\", \"processes\", \"registryKeyStates\", \"securityResources\",\n \"triggers\", \"userStates\", \"uriClickSecurityStates\", \"vulnerabilityStates\"\n },\n vendorInformation = {\n \"provider\", \"providerVersion\", \"subProvider\", \"vendor\", \n },\n userStates = {\n \"aadUserId\", \"accountName\", \"domainName\", \"emailRole\", \"isVpn\", \"logonDateTime\", \n \"logonId\", \"logonIp\", \"logonLocation\", \"logonType\", \"onPremisesSecurityIdentifier\",\n \"riskScore\", \"userAccountType\", \"userPrincipalName\"\n }\n}\n\nlocal MGMT_FIELD_ORDER = {\n [\"dlprulematch\"] = {\n message = {\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\",\n \"RecordType\",\n \"UserKey\",\n \"UserType\",\n \"Version\",\n \"Workload\",\n \"ObjectId\",\n \"UserId\",\n \"IncidentId\",\n \"PolicyDetails\",\n \"SensitiveInfoDetectionIsIncluded\",\n \"SharePointMetaData\",\n },\n PolicyDetails = {\n \"PolicyId\", \"PolicyName\", \"Rules\"\n },\n Rules = {\n \"ActionParameters\", \"Actions\", \"ConditionsMatched\", \"ManagementRuleId\", \"RuleId\", \"RuleMode\", \"RuleName\", \"Severity\"\n },\n ConditionsMatched = {\n \"ConditionMatchedInNewScheme\", \"OtherConditions\", \"SensitiveInformation\"\n },\n OtherConditions = {\n \"Name\", \"Value\"\n },\n SensitiveInformation = {\n \"ClassifierType\", \"Confidence\", \"Count\", \"SensitiveInformationDetailedClassificationAttributes\", \n \"SensitiveInformationDetections\", \"SensitiveInformationTypeName\", \"SensitiveType\"\n },\n SensitiveInformationDetailedClassificationAttributes = {\n \"Confidence\", \"Count\", \"IsMatch\"\n },\n SensitiveInformationDetections = {\n \"DetectedValues\", \"ResultsTruncated\"\n },\n DetectedValues = {\n \"Name\", \"Value\"\n },\n SharePointMetaData = {\n \"FileID\", \"FileName\", \"FileOwner\", \"FilePathUrl\", \"FileSize\", \"From\", \"IsViewableByExternalUsers\",\n \"IsVisibleOnlyToOdbOwner\", \"ItemCreationTime\", \"ItemLastModifiedTime\", \"ItemLastSharedTime\",\n \"SensitivityLabelIds\", \"SharedBy\", \"SiteAdmin\", \"SiteCollectionGuid\", \"SiteCollectionUrl\", \"UniqueID\"\n }\n },\n [\"dlpruleundo\"] = {\n message = {\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\",\n \"RecordType\",\n \"UserKey\",\n \"UserType\",\n \"Version\",\n \"Workload\",\n \"ObjectId\",\n \"UserId\",\n \"IncidentId\",\n \"PolicyDetails\",\n \"SensitiveInfoDetectionIsIncluded\",\n \"SharePointMetaData\",\n },\n ExceptionInfo = {\n \"Reason\"\n },\n PolicyDetails = {\n \"PolicyId\", \"PolicyName\", \"Rules\"\n },\n Rules = {\n \"ConditionsMatched\", \"ManagementRuleId\", \"OverriddenActions\", \"RuleId\", \"RuleMode\", \"RuleName\", \"Severity\"\n },\n ConditionsMatched = {\n \"ConditionMatchedInNewScheme\", \"SensitiveInformation\"\n },\n SensitiveInformation = {\n \"ClassifierType\", \"Confidence\", \"Count\", \"SensitiveInformationDetailedClassificationAttributes\", \n \"SensitiveInformationDetections\", \"SensitiveInformationTypeName\", \"SensitiveType\"\n },\n SensitiveInformationDetailedClassificationAttributes = {\n \"Confidence\", \"Count\", \"IsMatch\"\n },\n SensitiveInformationDetections = {\n \"DetectedValues\", \"ResultsTruncated\"\n },\n DetectedValues = {\n \"Name\", \"Value\"\n },\n SharePointMetaData = {\n \"FileID\", \"FileName\", \"FileOwner\", \"FilePathUrl\", \"FileSize\", \"From\", \"IsViewableByExternalUsers\",\n \"IsVisibleOnlyToOdbOwner\", \"ItemCreationTime\", \"ItemLastModifiedTime\", \"ItemLastSharedTime\",\n \"SensitivityLabelIds\", \"SharedBy\", \"SiteAdmin\", \"SiteCollectionGuid\", \"SiteCollectionUrl\", \"UniqueID\"\n }\n \n },\n [\"update user\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"userloggedin\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ClientIP\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"ActorIpAddress\", \"InterSystemsId\", \n \"IntraSystemId\", \"SupportTicketId\", \"Target\", \"TargetContextId\", \"ApplicationId\", \"DeviceProperties\", \"ErrorNumber\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n },\n DeviceProperties = {\n \"Name\", \"Value\"\n }\n },\n [\"add group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \"ExtendedProperties\",\n \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\", \"SupportTicketId\", \n \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"add member to group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"delete group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"update group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"remove member from group\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"add user\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"reset user password\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"delete user\"] = {\n message = {\n \"CreationTime\", \"Id\", \"Operation\", \"OrganizationId\", \"RecordType\", \"ResultStatus\", \"UserKey\", \n \"UserType\", \"Version\", \"Workload\", \"ObjectId\", \"UserId\", \"AzureActiveDirectoryEventType\", \n \"ExtendedProperties\", \"ModifiedProperties\", \"Actor\", \"ActorContextId\", \"InterSystemsId\", \"IntraSystemId\",\n \"SupportTicketId\", \"Target\", \"TargetContextId\"\n },\n ExtendedProperties = {\n \"Name\", \"Value\"\n },\n ModifiedProperties = {\n \"Name\", \"NewValue\", \"OldValue\"\n },\n Actor = {\n \"ID\", \"Type\"\n },\n Target = {\n \"ID\", \"Type\"\n }\n },\n [\"addedtogroup\"] = {\n message = {\n \"AppAccessContext\",\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\", \n \"RecordType\", \n \"UserKey\",\n \"UserType\", \n \"Version\", \n \"Workload\", \n \"ClientIP\", \n \"ObjectId\", \n \"UserId\", \n \"CorrelationId\", \n \"EventSource\", \n \"ItemType\", \n \"Site\", \n \"UserAgent\", \n \"WebId\", \n \"EventData\", \n \"TargetUserOrGroupType\", \n \"SiteUrl\", \n \"TargetUserOrGroupName\"\n },\n AppAccessContext = {\n \"AADSessionId\", \"CorrelationId\", \"UniqueTokenId\"\n },\n },\n [\"signinevent\"] = {\n message = {\n \"AppAccessContext\",\n \"CreationTime\",\n \"Id\",\n \"Operation\",\n \"OrganizationId\", \n \"RecordType\", \n \"UserKey\",\n \"UserType\", \n \"Version\", \n \"Workload\", \n \"ClientIP\", \n \"UserId\", \n \"AuthenticationType\",\n \"BrowserName\",\n \"BrowserVersion\",\n \"CorrelationId\",\n \"EventSource\",\n \"IsManagedDevice\",\n \"ItemType\",\n \"Platform\",\n \"UserAgent\",\n \"DeviceDisplayName\"\n },\n AppAccessContext = {\n \"AADSessionId\", \"CorrelationId\", \"UniqueTokenId\"\n },\n },\n}\n\nlocal function get_field_order(api, event_type)\n if api == \"Graph\" then\n return GRAPH_FIELD_ORDER or {}\n else\n local et = (event_type or \"\"):lower()\n return MGMT_FIELD_ORDER[et] or {}\n end\nend\n\nlocal IGNORE_KEYS = {\n _ob = true,\n timestamp = true, -- Ignore timestamp as we use start_time/end_time\n}\n\n-- Global entry point\nfunction processEvent(event)\n local e = event or {}\n -- If input already has OCSF-shaped dotted keys (post-mapped), just build nested and return\n if e[\"metadata.original_time\"] or e[\"category_uid\"] or e[\"class_uid\"] or e[\"dataSource.vendor\"] then\n return build_nested(e)\n end\n -- allow site id from event.site.id or event.site_id\n local site_id = nil\n local st = e[\"site\"]\n if type(st) == \"table\" and st[\"id\"] then site_id = st[\"id\"] elseif e[\"site_id\"] then site_id = e[\"site_id\"] end\n\n local api = detect_api(e)\n local working = cloneTable(e)\n\n if api == \"Graph\" then\n working = set_graph_synthetic_fields(working, site_id)\n local result = default_graph_api_parser(working)\n result.time = convertUtcToMilliseconds(e[\"createdDateTime\"])\n -- Store original input as JSON string in message field (with field ordering)\n local originalInput = deepCopy(event, IGNORE_KEYS) or {}\n result.message = encodeJson(originalInput, get_field_order(api, \"\"), \"message\")\n return result\n else\n working = set_mgmt_synthetic_fields(working, site_id)\n local result = default_mgmt_api_parser(working)\n result.time = convertUtcToMilliseconds(e[\"CreationTime\"])\n -- Store original input as JSON string in message field (with field ordering)\n local originalInput = deepCopy(event, IGNORE_KEYS) or {}\n local event_type = tostring(working[\"Operation\"] or \"Other\"):gsub(\"%.\", \"\")\n result.message = encodeJson(originalInput, get_field_order(api, event_type), \"message\")\n return result\n end\nend\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/microsoft_365/sample.json b/pipelines/community/transform_ocsf/microsoft_365/sample.json deleted file mode 100644 index defada4..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/sample.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "id": "cb1628d2-7c07-4435-9f4e-87fd44024d05", - "azureSubscriptionId": "6deb1e8b-896d-484a-a649-aeb6aebfeaf9", - "azureTenantId": "8c7e1716-01d8-4aeb-ac89-c691e8957e6f", - "activityGroupName": "Suspicious email forwarding", - "assignedTo": "unassigned", - "category": "MaliciousEmail", - "closedDateTime": "2026-04-20T13:40:52.634172Z", - "cloudAppStates": [ - { - "destinationServiceIp": "135.249.79.95", - "destinationServiceName": "SharePoint Online", - "riskScore": "66" - } - ], - "comments": [ - "Alert generated at 2026-04-20 03:33:52", - "Escalated to security team" - ], - "confidence": 97, - "createdDateTime": "2026-04-20T03:33:52.634172Z", - "description": "Microsoft 365 security alert: Suspicious email forwarding. This alert indicates potential maliciousemail activity detected in your environment.", - "detectionIds": [ - "de01dafc-efd2-47a5-9d71-9ca354ae2bc5" - ], - "eventDateTime": "2026-04-20T03:36:52.634172Z", - "feedback": "falsePositive", - "incidentIds": [], - "lastEventDateTime": "2026-04-20T03:54:52.634172Z", - "lastModifiedDateTime": "2026-04-20T03:40:52.634172Z", - "malwareStates": [], - "networkConnections": [ - { - "applicationName": "chrome.exe", - "destinationAddress": "199.227.89.52", - "destinationDomain": "graph.microsoft.com", - "destinationPort": "8080", - "destinationUrl": "https://domain-62.com/api/data", - "direction": "Outbound", - "domainRegisteredDateTime": "2025-07-18T03:40:52.634250Z", - "localDnsName": "workstation-937.internal.com", - "natDestinationAddress": "54.175.90.160", - "natDestinationPort": "41509", - "natSourceAddress": "10.79.101.231", - "natSourcePort": "40428", - "protocol": "UDP", - "riskScore": "24", - "sourceAddress": "10.246.169.54", - "sourcePort": "56074", - "status": "Timeout", - "urlCategory": "Malicious" - }, - { - "applicationName": "teams.exe", - "destinationAddress": "77.78.37.51", - "destinationDomain": "suspicious-domain.com", - "destinationPort": "993", - "destinationUrl": "https://suspicious-domain-9.com/api/data", - "direction": "Inbound", - "domainRegisteredDateTime": "2025-12-29T03:40:52.634286Z", - "localDnsName": "workstation-611.company.com", - "natDestinationAddress": "112.48.136.205", - "natDestinationPort": "13031", - "natSourceAddress": "10.81.153.186", - "natSourcePort": "62464", - "protocol": "ICMP", - "riskScore": "70", - "sourceAddress": "10.106.169.99", - "sourcePort": "24233", - "status": "Timeout", - "urlCategory": "Malicious" - }, - { - "applicationName": "outlook.exe", - "destinationAddress": "143.207.0.118", - "destinationDomain": "outlook.office365.com", - "destinationPort": "443", - "destinationUrl": "https://suspicious-domain-61.com/api/data", - "direction": "Outbound", - "domainRegisteredDateTime": "2026-04-02T03:40:52.634305Z", - "localDnsName": "workstation-181.internal.com", - "natDestinationAddress": "69.11.88.152", - "natDestinationPort": "30303", - "natSourceAddress": "10.117.172.211", - "natSourcePort": "16650", - "protocol": "ICMP", - "riskScore": "82", - "sourceAddress": "10.221.94.206", - "sourcePort": "37257", - "status": "Timeout", - "urlCategory": "Technology" - } - ], - "processes": [ - { - "accountName": "bob.jones", - "commandLine": "C:\\Windows\\System32\\regsvr32.exe -ExecutionPolicy Bypass 1c17ec96df224690", - "createdDateTime": "2026-04-20T03:15:52.634324Z", - "fileHash": { - "hashType": "sha256", - "hashValue": "ca810c03ed064d33ad983d1f289575f907a64d598cb04f709955f5ac1e617080" - }, - "integrityLevel": "Low", - "isElevated": "true", - "name": "regsvr32.exe", - "parentProcessCreatedDateTime": "2026-04-20T03:21:52.634333Z", - "parentProcessId": 8701, - "parentProcessName": "explorer.exe", - "path": "C:\\Windows\\System32\\regsvr32.exe", - "processId": 5212 - } - ], - "recommendedActions": [ - "Investigate user activity", - "Review mailbox rules", - "Check for additional indicators", - "Validate with user" - ], - "registryKeyStates": [], - "securityResources": [ - "Microsoft Cloud App Security", - "Azure Security Center", - "Microsoft Defender for Office 365" - ], - "severity": "critical", - "sourceMaterials": [ - "https://security.microsoft.com/alerts/f76fd6f1-7179-4045-a0ad-ca79d7ac572d", - "https://portal.office.com/adminportal/home#/MessageCenter/:/messages/cd845bfa-9e65-4051-896b-0b44d35b0cdd" - ], - "status": "dismissed", - "tags": [ - "maliciousemail", - "m365", - "security", - "automated" - ], - "title": "Microsoft 365 Alert: Suspicious email forwarding", - "triggers": [ - { - "name": "MaliciousEmail_detection_rule", - "type": "SecurityEvent", - "value": "79" - } - ], - "userStates": [ - { - "aadUserId": "5bdb7c45-546d-4f19-8b7c-b24815554c57", - "accountName": "jane.smith", - "domainName": "company.com", - "emailRole": "sender", - "isVpn": "true", - "logonDateTime": "2026-04-19T07:40:52.634379Z", - "logonId": "903143", - "logonIp": "89.36.216.146", - "logonLocation": "Seattle, WA", - "logonType": "Batch", - "onPremisesSecurityIdentifier": "S-1-5-21-797898851-168380601-9172-4847", - "riskScore": "29", - "userAgent": "Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.14326; Pro)", - "userPrincipalName": "jane.smith@company.com" - } - ], - "vendorInformation": { - "provider": "Microsoft", - "providerVersion": "1.0", - "subProvider": "Microsoft 365 Defender", - "vendor": "Microsoft" - }, - "riskScore": "75" -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/microsoft_365/serializer.lua b/pipelines/community/transform_ocsf/microsoft_365/serializer.lua deleted file mode 100644 index f4bfeda..0000000 --- a/pipelines/community/transform_ocsf/microsoft_365/serializer.lua +++ /dev/null @@ -1,1342 +0,0 @@ --- Microsoft 365 (O365) OCSF 1.0.0 parser (ported from Python Parsers/microsoft) - --- Helpers - -local function convertUtcToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - local year, month, day, hour, min, sec, frac = - string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)") - if not year then - return nil - end - local timeStruct = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - local localSeconds = os.time(timeStruct) - local utcDate = os.date("!*t", localSeconds) - local adjustedSeconds - if utcDate.year == timeStruct.year and utcDate.month == timeStruct.month and - utcDate.day == timeStruct.day and utcDate.hour == timeStruct.hour and - utcDate.min == timeStruct.min and utcDate.sec == timeStruct.sec then - adjustedSeconds = localSeconds - else - local utcSeconds = os.time(utcDate) - adjustedSeconds = localSeconds + (localSeconds - utcSeconds) - end - local milli = 0 - if frac and frac ~= "" then - milli = tonumber((frac .. "000"):sub(1, 3)) - end - return adjustedSeconds * 1000 + milli -end - -local function split(str, delimiter) - local result = {} - local s = tostring(str or "") - local escaped = delimiter:gsub("[%.%+%*%?%^%$%(%)%[%]%%]", "%%%1") - local pattern = "([^" .. escaped .. "]+)" - for token in s:gmatch(pattern) do table.insert(result, token) end - if #result == 0 and #s > 0 then table.insert(result, s) end - return result -end - -local function getByPath(obj, keys) - local cur = obj - for _, k in ipairs(keys) do - if cur ~= nil and type(cur) == "table" then cur = cur[k] else return nil end - end - return cur -end - -local function setByDottedPath(t, dotted, value) - local keys = split(dotted, ".") - local cur = t - for i=1,#keys-1 do - local k = keys[i] - if type(cur[k]) ~= "table" then cur[k] = {} end - cur = cur[k] - end - cur[keys[#keys]] = value -end - -local function flatten_table(tbl, prefix, out) - out = out or {} - prefix = prefix or "" - - for k, v in pairs(tbl) do - local key - - -- ARRAY HANDLING (0-based index) - if type(k) == "number" then - key = string.format("%s[%d]", prefix, k - 1) - - -- NORMAL OBJECT KEY - else - if prefix ~= "" then - key = prefix .. "." .. k - else - key = k - end - end - - -- recurse - if type(v) == "table" then - flatten_table(v, key, out) - else - out[key] = v - end - end - - return out -end - -local function apply_mapping(event, mapping) - local out = {} - - -- normal mapping (existing behaviour) - for src, dst in pairs(mapping) do - local val = getByPath(event, split(src, ".")) - if val ~= nil then out[dst] = val end - end - - -- NEW: auto-capture unmapped fields - local flat_event = flatten_table(event) - for src, val in pairs(flat_event) do - if mapping[src] == nil then - out["unmapped." .. src] = val - end - end - - return out -end - -local function build_nested(flat) - local nested = {} - for k, v in pairs(flat) do setByDottedPath(nested, k, v) end - return nested -end - -local function cloneTable(src) - local dst = {} - for k,v in pairs(src or {}) do dst[k] = v end - return dst -end - --- Simple email detection -local function is_email(s) - if type(s) ~= "string" then return false end - return s:find("@") ~= nil -end - --- Common + synthetic field helpers -local function get_status_default_ocsf_mapping(status) - if status == "Success" then return 1 - elseif status == "Failure" then return 2 - else return 99 end -end - -local function get_device_os_type(platform) - local mapping = { Win = 100, Android = 201, ["iOS"] = 301 } - if type(platform) ~= "string" then return 99 end - return mapping[platform] or 99 -end - -local function get_actor_type_details(user_type) - local map = { - [0] = { type = "User", id = 1 }, - [2] = { type = "Admin", id = 2 }, - [4] = { type = "System", id = 4 }, - } - local rec = map[user_type] - if rec then return rec.type, rec.id else return "Other", 99 end -end - -local function get_state_id(event_type) - if type(event_type) == "string" and event_type:lower() == "dlprulematch" then return 1 else return 0 end -end - -local function extract_policy_details(log) - local analytic_id, analytic_name = nil, nil - local pd = log["PolicyDetails"] - if type(pd) == "table" and #pd > 0 and type(pd[1]) == "table" then - if pd[1]["PolicyId"] then analytic_id = pd[1]["PolicyId"] end - if pd[1]["PolicyName"] then analytic_name = pd[1]["PolicyName"] end - end - return analytic_id, analytic_name -end - -local function get_device_property(log) - local device_props = log["DeviceProperties"] - if type(device_props) == "table" then - for _, dp in ipairs(device_props) do - if type(dp) == "table" then - if dp["Name"] == "OS" then - log["device_os_type"] = dp["Value"] - log["device_os_type_id"] = (dp["Value"] == "Linux" and 200) or (dp["Value"] == "Windows" and 100) or 99 - elseif dp["Name"] == "SessionId" then - log["actor_session_uid"] = dp["Value"] - elseif dp["Name"] == "IsCompliantAndManaged" then - log["device_is_compliant"] = dp["Value"] - log["device_is_managed"] = dp["Value"] - end - end - end - end - return log -end - -local function get_resource_data(log) - local out = {} - local mprops = log["ModifiedProperties"] - if type(mprops) == "table" then - for _, obj in ipairs(mprops) do - if type(obj) == "table" and obj["Name"] then - out[obj["Name"]] = { NewValue = obj["NewValue"], OldValue = obj["OldValue"] } - end - end - end - return out -end - -local function get_mgmt_observables(log) - local obs = {} - local sp = log["SharePointMetaData"] - if type(sp) == "table" then - if sp["FileName"] then table.insert(obs, { type_id = 7, type = "FileName", name = "process.file.name", value = sp["FileName"] }) end - if sp["From"] then table.insert(obs, { type_id = 4, type = "User Name", name = "unmapped.sharePointMetaData.from", value = sp["From"] }) end - end - return obs -end - --- Synthetic for Graph API -local function set_graph_synthetic_fields(log, site_id) - log["category_uid"] = 2 - log["class_uid"] = 2001 - log["class_name"] = "Security Finding" - log["type_uid"] = 200199 - log["type_name"] = "Security Finding: Other" - log["OCSF_version"] = "1.0.0" - log["category_name"] = "Findings" - log["activity_id"] = 99 - log["dataSource"] = { name = "Microsoft O365", category = "security", vendor = "Microsoft" } - log["event"] = { type = log["status"] } - log["cloud"] = { provider = "Microsoft Azure" } - if site_id then log["site"] = { id = site_id } end - -- observables - local observables = {} - if log["azureTenantId"] then table.insert(observables, { type_id = 10, type = "Resource UID", name = "unmapped.azureTenantId", value = log["azureTenantId"] }) end - log["observables"] = observables - if type(log["sourceMaterials"]) == "table" then log["sourceMaterials"] = log["sourceMaterials"][1] end - return log -end - --- Severity for Mgmt -local function get_severity_id(event_type) - local m = { - dlprulematch = 1, dlpruleundo = 0, ["update user"] = 0, userloggedin = 0, ["add group"] = 0, - ["add member to group"] = 0, ["delete group"] = 0, ["update group"] = 0, ["remove member from group"] = 0, - ["add user"] = 0, ["reset user password"] = 0, ["delete user"] = 0, addedtogroup = 0, signinevent = 0, - } - return m[(event_type or ""):lower()] or 0 -end - -local function get_mgmt_log_mapping_fields(event_type) - local map = { - ["dlprulematch"] = { - activity = { name = "Create", id = 99 }, class = { name = "Security Finding", id = 2001 }, category = { name = "Findings", id = 2 }, - }, - ["dlpruleundo"] = { - activity = { name = "DLPRuleUndo", id = 99 }, class = { name = "Security Finding", id = 2001 }, category = { name = "Findings", id = 2 }, - }, - ["update user"] = { - activity = { name = "Update user", id = 99 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["userloggedin"] = { - activity = { name = "Logon", id = 1 }, class = { name = "Authentication", id = 3002 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["add group"] = { - activity = { name = "Add group", id = 99 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["add member to group"] = { - activity = { name = "Add User", id = 3 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["delete group"] = { - activity = { name = "Delete group", id = 99 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["update group"] = { - activity = { name = "Update group.", id = 99 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["remove member from group"] = { - activity = { name = "Remove User", id = 4 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["add user"] = { - activity = { name = "Create", id = 1 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["reset user password"] = { - activity = { name = "Password Reset", id = 4 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["delete user"] = { - activity = { name = "Delete", id = 6 }, class = { name = "Account Change", id = 3001 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["addedtogroup"] = { - activity = { name = "Add User", id = 3 }, class = { name = "Group Management", id = 3006 }, category = { name = "Identity & Access Management", id = 3 }, - }, - ["signinevent"] = { - activity = { name = "Logon", id = 1 }, class = { name = "Authentication", id = 3002 }, category = { name = "Identity & Access Management", id = 3 }, - }, - } - local m = map[(event_type or ""):lower()] or { activity = { name = "Other", id = 99 }, class = { name = "Base Event", id = 0}, category = { name = "Uncategorized", id = 0 } } - local class_name, class_id = m.class.name, m.class.id - local activity_name, activity_id = m.activity.name, m.activity.id - local type_name, type_id = string.format("%s: %s", class_name, activity_name), (class_id * 100) + activity_id - local category_name, category_id = m.category.name, m.category.id - return activity_name, activity_id, class_name, class_id, type_name, type_id, category_name, category_id -end - --- Mapping tables from python -local function get_default_graph_ocsf_mapping() - return { - ["activityGroupName"] = "resources.group_name", - ["category"] = "finding.types", - ["closedDateTime"] = "end_time_dt", - ["confidence"] = "confidence", - ["createdDateTime"] = "metadata.original_time", - ["eventDateTime"] = "start_time_dt", - ["id"] = "metadata.uid", - ["lastModifiedDateTime"] = "finding.modified_time", - ["sourceMaterials"] = "finding.src_url", - ["status"] = "activity_name", - ["vendorInformation.provider"] = "metadata.product.name", - ["vendorInformation.providerVersion"] = "metadata.product.version", - ["vendorInformation.vendor"] = "metadata.product.vendor_name", - ["lastEventDateTime"] = "end_time", - ["title"] = "finding.title", - ["category_uid"] = "category_uid", - ["class_uid"] = "class_uid", - ["type_uid"] = "type_uid", - ["category_name"] = "category_name", - ["class_name"] = "class_name", - ["type_name"] = "type_name", - ["activity_id"] = "activity_id", - ["cloud.provider"] = "cloud.provider", - ["OCSF_version"] = "metadata.version", - ["observables"] = "observables", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["event.type"] = "event.type", - ["dataSource.name"] = "dataSource.name", - ["dataSource.vendor"] = "dataSource.vendor", - } -end - -local function generic_mgmt_mapping() - return { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - } -end - -local function get_mgmt_default_mapping(event_type) - -- ported from python OCSFMapping.get_mgmt_default_mapping - local default = { - ["dlprulematch"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "finding.title", - ["OrganizationId"] = "cloud.org.uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["IncidentId"] = "finding.uid", - ["SharePointMetaData.FileID"] = "process.file.uid", - ["SharePointMetaData.FileName"] = "process.file.name", - ["SharePointMetaData.FileOwner"] = "process.file.owner.uid", - ["SharePointMetaData.FilePathUrl"] = "process.file.path", - ["SharePointMetaData.FileSize"] = "process.file.size", - ["SharePointMetaData.From"] = "resources.data.from", - ["SharePointMetaData.IsViewableByExternalUsers"] = "resources.data.is_viewable_by_external_users", - ["SharePointMetaData.IsVisibleOnlyToOdbOwner"] = "resources.data.is_visible_only_to_Odb_owner", - ["SharePointMetaData.ItemCreationTime"] = "finding.created_time", - ["SharePointMetaData.ItemLastModifiedTime"] = "finding.modified_time", - ["SharePointMetaData.ItemLastSharedTime"] = "resources.data.item_last_shared_time", - ["SharePointMetaData.SensitivityLabelIds"] = "resources.data.sensitivity_label_ids", - ["SharePointMetaData.SharedBy"] = "resources.data.shared_by", - ["SharePointMetaData.SiteAdmin"] = "resources.data.site_admin", - ["SharePointMetaData.SiteCollectionGuid"] = "resources.data.site_collection_guid", - ["SharePointMetaData.SiteCollectionUrl"] = "resources.data.site_collection_url", - ["SharePointMetaData.UniqueID"] = "resources.data.unique_id", - ["state_id"] = "state_id", - ["status_id"] = "status_id", - ["analytic.uid"] = "analytic.uid", - ["analytic.name"] = "analytic.name", - ["analytic_type_id"] = "analytic.type_id", - ["process_file_owner_type_id"] = "process.file.owner.type_id", - }, - ["dlpruleundo"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "finding.title", - ["OrganizationId"] = "cloud.org.uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["IncidentId"] = "finding.uid", - ["SharePointMetaData.FileID"] = "process.file.uid", - ["SharePointMetaData.FileName"] = "process.file.name", - ["SharePointMetaData.FileOwner"] = "process.file.owner.uid", - ["SharePointMetaData.FilePathUrl"] = "process.file.path", - ["SharePointMetaData.FileSize"] = "process.file.size", - ["SharePointMetaData.From"] = "resources.data.from", - ["SharePointMetaData.IsViewableByExternalUsers"] = "resources.data.is_viewable_by_external_users", - ["SharePointMetaData.IsVisibleOnlyToOdbOwner"] = "resources.data.is_visible_only_to_Odb_owner", - ["SharePointMetaData.ItemCreationTime"] = "finding.created_time", - ["SharePointMetaData.ItemLastModifiedTime"] = "finding.modified_time", - ["SharePointMetaData.ItemLastSharedTime"] = "resources.data.item_last_shared_time", - ["SharePointMetaData.SensitivityLabelIds"] = "resources.data.sensitivity_label_ids", - ["SharePointMetaData.SharedBy"] = "resources.data.shared_by", - ["SharePointMetaData.SiteAdmin"] = "resources.data.site_admin", - ["SharePointMetaData.SiteCollectionGuid"] = "resources.data.site_collection_guid", - ["SharePointMetaData.SiteCollectionUrl"] = "resources.data.site_collection_url", - ["SharePointMetaData.UniqueID"] = "resources.data.unique_id", - ["analytic.uid"] = "analytic.uid", - ["analytic.name"] = "analytic.name", - ["analytic_type_id"] = "analytic.type_id", - ["process_file_owner_type_id"] = "process.file.owner.type_id", - ["state_id"] = "state_id", - }, - ["update user"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "activity_name", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["userloggedin"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ClientIP"] = "device.ip", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["ActorIpAddress"] = "src_endpoint.ip", - ["InterSystemsId"] = "metadata.correlation_uid", - ["ApplicationId"] = "dst_endpoint.uid", - ["ErrorNumber"] = "api.response.code", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["device_os_type"] = "device.os.type", - ["device_os_type_id"] = "device.os.type_id", - ["actor_session_uid"] = "actor.session.uid", - ["device_is_compliant"] = "device.is_compliant", - ["device_is_managed"] = "device.is_managed", - }, - ["add group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "activity_name", - ["OrganizationId"] = "cloud.org.uid", - ["ResultStatus"] = "status_detail", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["add member to group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["ResultStatus"] = "status_detail", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ActorContextId"] = "actor.user.org.uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["delete group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["update group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["Operation"] = "activity_name", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["remove member from group"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["resource.data"] = "resource.data", - }, - ["add user"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["reset user password"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["delete user"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.UserKey", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ResultStatus"] = "status_detail", - ["ActorContextId"] = "actor.user.org.uid", - ["InterSystemsId"] = "metadata.correlation_uid", - ["TargetContextId"] = "user.org.uid", - ["status_id"] = "status_id", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - }, - ["addedtogroup"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ClientIP"] = "device.ip", - ["CorrelationId"] = "metadata.correlation_uid", - ["EventSource"] = "resource.name", - ["EventData"] = "status_detail", - ["TargetUserOrGroupType"] = "user.groups.type", - ["TargetUserOrGroupName"] = "user.groups.name", - ["AppAccessContext.CorrelationId"] = "actor.idp.uid", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - }, - ["signinevent"] = { - ["CreationTime"] = "metadata.original_time", - ["Id"] = "metadata.uid", - ["OrganizationId"] = "cloud.org.uid", - ["UserKey"] = "actor.user.credential_uid", - ["Version"] = "metadata.product.version", - ["Workload"] = "metadata.product.name", - ["ClientIP"] = "src_endpoint.ip", - ["CorrelationId"] = "metadata.correlation_uid", - ["EventSource"] = "service.name", - ["IsManagedDevice"] = "device.is_managed", - ["Platform"] = "device.os.type", - ["UserAgent"] = "http_request.user_agent", - ["DeviceDisplayName"] = "device.ip", - ["AppAccessContext.AADSessionId"] = "actor.session.uid", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["device_os_type_id"] = "device.os.type_id", - }, - } - local et = (event_type or ""):lower() - return default[et] or generic_mgmt_mapping() -end - -local function common_mapping() - return { - ["category_name"] = "category_name", - ["category_uid"] = "category_uid", - ["class_uid"] = "class_uid", - ["severity_id"] = "severity_id", - ["activity_name"] = "activity_name", - ["activity_id"] = "activity_id", - ["type_uid"] = "type_uid", - ["product_vendor_name"] = "metadata.product.vendor_name", - ["product_name"] = "metadata.product.name", - ["OCSF_version"] = "metadata.version", - ["observables"] = "observables", - ["dataSource.category"] = "dataSource.category", - ["site.id"] = "site.id", - ["event.type"] = "event.type", - ["dataSource.name"] = "dataSource.name", - ["dataSource.vendor"] = "dataSource.vendor", - ["message"] = "message", - ["class_name"] = "class_name", - ["type_name"] = "type_name", - ["actor_user_email_addr"] = "actor.user.email_addr", - ["actor_user_uid"] = "actor.user.uid", - ["actor_user_type_id"] = "actor.user.type_id", - ["actor_user_type"] = "actor.user.type", - ["user_email_addr"] = "user.email_addr", - ["user_uid"] = "user.uid", - ["user_type_id"] = "user.type_id", - ["user_type"] = "user.type", - ["status_id"] = "status_id", - } -end - --- Mgmt synthetic enrichment -local function set_mgmt_synthetic_fields(log, site_id) - log["product_vendor_name"] = "Microsoft" - log["OCSF_version"] = "1.0.0" - local event_type = tostring(log["Operation"] or "Other") - local event_type_dup = event_type:gsub("%.", "") - - log["severity_id"] = get_severity_id(event_type_dup) - local activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid = get_mgmt_log_mapping_fields(event_type_dup) - log["activity_name"], log["activity_id"], log["class_name"], log["class_uid"], log["type_name"], log["type_uid"], log["category_name"], log["category_uid"] = activity_name, activity_id, class_name, class_uid, type_name, type_uid, category_name, category_uid - - log["status_id"] = get_status_default_ocsf_mapping(log["ResultStatus"]) - - if site_id then log["site"] = { id = site_id } end - - if log["activity_id"] == 99 then - log["event"] = { type = event_type } - log["activity_name"] = event_type - else - log["event"] = { type = log["activity_name"] } - end - - log["dataSource"] = { name = "Microsoft O365", category = "security", vendor = "Microsoft" } - - if event_type_dup:lower() == "dlprulematch" or event_type_dup:lower() == "dlpruleundo" then - log["analytic_type_id"] = 1 - log["process_file_owner_type_id"] = 99 - log["observables"] = get_mgmt_observables(log) - log["state_id"] = get_state_id(event_type_dup) - local aid, aname = extract_policy_details(log) - log["analytic"] = { uid = aid, name = aname } - else - -- set actor values - if (event_type_dup:lower() == "userloggedin") then - log["actor_user_type"], log["actor_user_type_id"] = "User", 1 - end - local userId = log["UserId"] - if is_email(userId) then - log["actor_user_email_addr"] = userId - elseif type(userId) == "string" and userId ~= "" then - log["actor_user_uid"] = userId - end - local actors = log["Actor"] - if type(actors) == "table" and #actors > 0 then - for _, a in ipairs(actors) do - if type(a) == "table" then - local id = a["ID"] - if is_email(id) then - log["actor_user_email_addr"] = id - local t, tid = get_actor_type_details(a["Type"]) - log["actor_user_type"], log["actor_user_type_id"] = t, tid - break - elseif type(id) == "string" and id:match("-") then - log["actor_user_uid"] = id - local t, tid = get_actor_type_details(a["Type"]) - log["actor_user_type"], log["actor_user_type_id"] = t, tid - break - end - end - end - else - local t, tid = get_actor_type_details(log["UserType"]) - log["actor_user_type"], log["actor_user_type_id"] = t, tid - end - - -- set user target data - local target = log["Target"] - if type(target) == "table" and #target > 0 and type(target[1]) == "table" then - local tid = target[1]["ID"] - if is_email(tid) then log["user_email_addr"] = tid else log["user_uid"] = tid end - local t, tidv = get_actor_type_details(target[1]["Type"]) - log["user_type"], log["user_type_id"] = t, tidv - end - end - - if log["UserKey"] == "Not Available" or log["UserKey"] == "NA" then log["UserKey"] = "" end - - local etl = event_type_dup:lower() - if etl == "add group" or etl == "add member to group" or etl == "delete group" or etl == "update group" or etl == "remove member from group" then - log["resource"] = { data = get_resource_data(log) } - end - - if etl == "signinevent" then - log["device_os_type_id"] = get_device_os_type(log["Platform"]) - end - - if etl == "userloggedin" then - log = get_device_property(log) - end - - return log -end - --- Parsers -local function default_graph_api_parser(alertLog) - return build_nested(apply_mapping(alertLog, get_default_graph_ocsf_mapping())) -end - -local function default_mgmt_api_parser(log) - local event_type = tostring(log["Operation"] or "Other"):gsub("%.", "") - local mapping = get_mgmt_default_mapping(event_type) - -- merge with common mapping - for src, dst in pairs(common_mapping()) do mapping[src] = dst end - local flat = apply_mapping(log, mapping) - return build_nested(flat) -end - --- API detection: Graph has createdDateTime, vendorInformation; Mgmt has CreationTime/Operation -local function detect_api(event) - if event["createdDateTime"] or getByPath(event, {"vendorInformation"}) then return "Graph" end - return "Mgmt" -end - --- Helper: Encode Lua table to JSON string with field ordering -local function encodeJson(obj, fieldOrder, key) - if obj == nil then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - if isArray then - local items = {} - for i = 1, maxIndex do - table.insert(items, encodeJson(obj[i], fieldOrder, key)) - end - return "[" .. table.concat(items, ",") .. "]" - else - local items = {} - local fieldOrdering = fieldOrder[key] or {} - - -- Phase 1: ordered keys - for _, fieldName in ipairs(fieldOrdering) do - local v = obj[fieldName] - if v ~= nil then - local encoded = encodeJson(v, fieldOrder, fieldName) - if encoded ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - -- Phase 2: remaining keys (not in fieldOrder) - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrdering) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local encoded = encodeJson(v, fieldOrder, keyStr) - if encoded ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '":' .. encoded) - end - end - end - - return "{" .. table.concat(items, ",") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - -local function deepCopy(value, ignoreKeys) - if type(value) ~= "table" then - return value - end - local copy = {} - for k, v in pairs(value) do - if not (ignoreKeys and ignoreKeys[k]) then - copy[k] = deepCopy(v, ignoreKeys) - end - end - return copy -end - -local GRAPH_FIELD_ORDER = { - message = { - "id", "azureTenantId", "azureSubscriptionId", "riskScore", "tags", "activityGroupName", "assignedTo", - "category", "closedDateTime", "comments", "confidence", "createdDateTime", "description", "detectionIds", - "eventDateTime", "feedback", "incidentIds", "lastEventDateTime", "lastModifiedDateTime", "recommendedActions", - "severity", "sourceMaterials", "status", "title", "vendorInformation", "alertDetections", - "cloudAppStates", "fileStates", "hostStates", "historyStates", "investigationSecurityStates", - "malwareStates", "messageSecurityStates", "networkConnections", "processes", "registryKeyStates", "securityResources", - "triggers", "userStates", "uriClickSecurityStates", "vulnerabilityStates" - }, - vendorInformation = { - "provider", "providerVersion", "subProvider", "vendor", - }, - userStates = { - "aadUserId", "accountName", "domainName", "emailRole", "isVpn", "logonDateTime", - "logonId", "logonIp", "logonLocation", "logonType", "onPremisesSecurityIdentifier", - "riskScore", "userAccountType", "userPrincipalName" - } -} - -local MGMT_FIELD_ORDER = { - ["dlprulematch"] = { - message = { - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ObjectId", - "UserId", - "IncidentId", - "PolicyDetails", - "SensitiveInfoDetectionIsIncluded", - "SharePointMetaData", - }, - PolicyDetails = { - "PolicyId", "PolicyName", "Rules" - }, - Rules = { - "ActionParameters", "Actions", "ConditionsMatched", "ManagementRuleId", "RuleId", "RuleMode", "RuleName", "Severity" - }, - ConditionsMatched = { - "ConditionMatchedInNewScheme", "OtherConditions", "SensitiveInformation" - }, - OtherConditions = { - "Name", "Value" - }, - SensitiveInformation = { - "ClassifierType", "Confidence", "Count", "SensitiveInformationDetailedClassificationAttributes", - "SensitiveInformationDetections", "SensitiveInformationTypeName", "SensitiveType" - }, - SensitiveInformationDetailedClassificationAttributes = { - "Confidence", "Count", "IsMatch" - }, - SensitiveInformationDetections = { - "DetectedValues", "ResultsTruncated" - }, - DetectedValues = { - "Name", "Value" - }, - SharePointMetaData = { - "FileID", "FileName", "FileOwner", "FilePathUrl", "FileSize", "From", "IsViewableByExternalUsers", - "IsVisibleOnlyToOdbOwner", "ItemCreationTime", "ItemLastModifiedTime", "ItemLastSharedTime", - "SensitivityLabelIds", "SharedBy", "SiteAdmin", "SiteCollectionGuid", "SiteCollectionUrl", "UniqueID" - } - }, - ["dlpruleundo"] = { - message = { - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ObjectId", - "UserId", - "IncidentId", - "PolicyDetails", - "SensitiveInfoDetectionIsIncluded", - "SharePointMetaData", - }, - ExceptionInfo = { - "Reason" - }, - PolicyDetails = { - "PolicyId", "PolicyName", "Rules" - }, - Rules = { - "ConditionsMatched", "ManagementRuleId", "OverriddenActions", "RuleId", "RuleMode", "RuleName", "Severity" - }, - ConditionsMatched = { - "ConditionMatchedInNewScheme", "SensitiveInformation" - }, - SensitiveInformation = { - "ClassifierType", "Confidence", "Count", "SensitiveInformationDetailedClassificationAttributes", - "SensitiveInformationDetections", "SensitiveInformationTypeName", "SensitiveType" - }, - SensitiveInformationDetailedClassificationAttributes = { - "Confidence", "Count", "IsMatch" - }, - SensitiveInformationDetections = { - "DetectedValues", "ResultsTruncated" - }, - DetectedValues = { - "Name", "Value" - }, - SharePointMetaData = { - "FileID", "FileName", "FileOwner", "FilePathUrl", "FileSize", "From", "IsViewableByExternalUsers", - "IsVisibleOnlyToOdbOwner", "ItemCreationTime", "ItemLastModifiedTime", "ItemLastSharedTime", - "SensitivityLabelIds", "SharedBy", "SiteAdmin", "SiteCollectionGuid", "SiteCollectionUrl", "UniqueID" - } - - }, - ["update user"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["userloggedin"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ClientIP", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "ActorIpAddress", "InterSystemsId", - "IntraSystemId", "SupportTicketId", "Target", "TargetContextId", "ApplicationId", "DeviceProperties", "ErrorNumber" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - }, - DeviceProperties = { - "Name", "Value" - } - }, - ["add group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", "ExtendedProperties", - "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", "SupportTicketId", - "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["add member to group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["delete group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["update group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["remove member from group"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["add user"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["reset user password"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["delete user"] = { - message = { - "CreationTime", "Id", "Operation", "OrganizationId", "RecordType", "ResultStatus", "UserKey", - "UserType", "Version", "Workload", "ObjectId", "UserId", "AzureActiveDirectoryEventType", - "ExtendedProperties", "ModifiedProperties", "Actor", "ActorContextId", "InterSystemsId", "IntraSystemId", - "SupportTicketId", "Target", "TargetContextId" - }, - ExtendedProperties = { - "Name", "Value" - }, - ModifiedProperties = { - "Name", "NewValue", "OldValue" - }, - Actor = { - "ID", "Type" - }, - Target = { - "ID", "Type" - } - }, - ["addedtogroup"] = { - message = { - "AppAccessContext", - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ClientIP", - "ObjectId", - "UserId", - "CorrelationId", - "EventSource", - "ItemType", - "Site", - "UserAgent", - "WebId", - "EventData", - "TargetUserOrGroupType", - "SiteUrl", - "TargetUserOrGroupName" - }, - AppAccessContext = { - "AADSessionId", "CorrelationId", "UniqueTokenId" - }, - }, - ["signinevent"] = { - message = { - "AppAccessContext", - "CreationTime", - "Id", - "Operation", - "OrganizationId", - "RecordType", - "UserKey", - "UserType", - "Version", - "Workload", - "ClientIP", - "UserId", - "AuthenticationType", - "BrowserName", - "BrowserVersion", - "CorrelationId", - "EventSource", - "IsManagedDevice", - "ItemType", - "Platform", - "UserAgent", - "DeviceDisplayName" - }, - AppAccessContext = { - "AADSessionId", "CorrelationId", "UniqueTokenId" - }, - }, -} - -local function get_field_order(api, event_type) - if api == "Graph" then - return GRAPH_FIELD_ORDER or {} - else - local et = (event_type or ""):lower() - return MGMT_FIELD_ORDER[et] or {} - end -end - -local IGNORE_KEYS = { - _ob = true, - timestamp = true, -- Ignore timestamp as we use start_time/end_time -} - --- Global entry point -function processEvent(event) - local e = event or {} - -- If input already has OCSF-shaped dotted keys (post-mapped), just build nested and return - if e["metadata.original_time"] or e["category_uid"] or e["class_uid"] or e["dataSource.vendor"] then - return build_nested(e) - end - -- allow site id from event.site.id or event.site_id - local site_id = nil - local st = e["site"] - if type(st) == "table" and st["id"] then site_id = st["id"] elseif e["site_id"] then site_id = e["site_id"] end - - local api = detect_api(e) - local working = cloneTable(e) - - if api == "Graph" then - working = set_graph_synthetic_fields(working, site_id) - local result = default_graph_api_parser(working) - result.time = convertUtcToMilliseconds(e["createdDateTime"]) - -- Store original input as JSON string in message field (with field ordering) - local originalInput = deepCopy(event, IGNORE_KEYS) or {} - result.message = encodeJson(originalInput, get_field_order(api, ""), "message") - return result - else - working = set_mgmt_synthetic_fields(working, site_id) - local result = default_mgmt_api_parser(working) - result.time = convertUtcToMilliseconds(e["CreationTime"]) - -- Store original input as JSON string in message field (with field ordering) - local originalInput = deepCopy(event, IGNORE_KEYS) or {} - local event_type = tostring(working["Operation"] or "Other"):gsub("%.", "") - result.message = encodeJson(originalInput, get_field_order(api, event_type), "message") - return result - end -end diff --git a/pipelines/community/transform_ocsf/okta/metadata.yaml b/pipelines/community/transform_ocsf/okta/metadata.yaml deleted file mode 100644 index 617525c..0000000 --- a/pipelines/community/transform_ocsf/okta/metadata.yaml +++ /dev/null @@ -1,53 +0,0 @@ -grade: - letter: F - score: 45 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Okta. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: okta - dataSource: Okta - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "API Key & Secret" - sample_record: "{\n \"actor\": {\n \"id\": \"00u4729hjsVRU197Y5d7\",\n \"type\": \"User\",\n\ - \ \"alternateId\": \"pnnpydhb@example.com\",\n \"displayName\": \"Pnnpydhb\",\n \"detailEntry\"\ - : null\n },\n \"client\": {\n \"userAgent\": {\n \"rawUserAgent\": \"Mozilla/5.0 (X11; Linux\ - \ x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36\",\n \"os\":\ - \ \"Linux\",\n \"browser\": \"CHROME\"\n },\n \"zone\": \"null\",\n \"device\": \"Computer\"\ - ,\n \"id\": null,\n \"ipAddress\": \"95.196.9.145\",\n \"geographicalContext\": {\n \ - \ \"city\": \"Hyderabad\",\n \"state\": \"Telangana\",\n \"country\": \"India\",\n \ - \ \"postalCode\": \"500004\",\n \"geolocation\": {\n \"lat\": 17.3724,\n \"lon\"\ - : 78.4378\n }\n }\n },\n \"device\": null,\n \"authenticationContext\": {\n \"authenticationProvider\"\ - : null,\n \"credentialProvider\": null,\n \"credentialType\": null,\n \"issuer\": null,\n\ - \ \"interface\": null,\n \"authenticationStep\": 0,\n \"rootSessionId\": \"102eA1R\",\n \ - \ \"externalSessionId\": \"102eA1R2M\"\n },\n \"displayMessage\": \"Create API token\",\n \"\ - eventType\": \"system.api_token.create\",\n \"outcome\": {\n \"result\": \"SUCCESS\",\n \"\ - reason\": null\n },\n \"published\": \"2026-04-20T02:26:37.000Z\",\n \"securityContext\": {\n \ - \ \"asNumber\": 150008,\n \"asOrg\": \"s r fibernet\",\n \"isp\": \"pioneer elabs ltd.\",\n\ - \ \"domain\": null,\n \"isProxy\": false\n },\n \"severity\": \"OTHER\",\n \"debugContext\"\ - : {\n \"debugData\": {\n \"concurrencyPercentage\": \"50\",\n \"requestId\": \"923c54e8-a3b4-4518-bc64-00b228adbe84\"\ - ," - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: okta, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: F - harness_score: 45 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/okta/okta.json b/pipelines/community/transform_ocsf/okta/okta.json deleted file mode 100644 index 571b587..0000000 --- a/pipelines/community/transform_ocsf/okta/okta.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Okta", - "grade": { - "letter": "F", - "score": 45, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Okta. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "okta", - "source_name": "okta", - "version": "1.0.0", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "okta-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "\n-- Lua implementation for Okta OCSF 1.0.0 schema\n\n-- Helper function to safely access nested dictionary keys\nfunction safelyAccessNestedDictKeys(keys, dictObject)\n local current = dictObject\n for _, key in ipairs(keys) do\n if current and type(current) == \"table\" then\n current = current[key]\n else\n return nil\n end\n end\n return current\nend\n\n-- Helper function to split string by delimiter\nfunction split(str, delimiter)\n local result = {}\n -- Escape special regex characters in delimiter\n local escapedDelimiter = delimiter:gsub(\"[%.%+%*%?%^%$%(%)%[%]%%]\", \"%%%1\")\n local pattern = \"([^\" .. escapedDelimiter .. \"]+)\"\n\n for token in str:gmatch(pattern) do\n table.insert(result, token)\n end\n\n -- Handle empty string case\n if #result == 0 and #str > 0 then\n table.insert(result, str)\n end\n\n return result\nend\n\n-- Helper function to convert timestamp to milliseconds\nfunction convertToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n \n -- Parse ISO 8601 timestamp (e.g., \"2023-04-24T04:55:30.535Z\")\n local year, month, day, hour, min, sec, ms = timestamp:match(\"(%d%d%d%d)-(%d%d)-(%d%d)T(%d%d):(%d%d):(%d%d)%.(%d%d%d)Z\")\n \n if year and month and day and hour and min and sec and ms then\n -- Convert to milliseconds since epoch\n local time = os.time({\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec)\n })\n return (time * 1000) + tonumber(ms)\n end\n \n return nil\nend\n\n-- Ordered JSON encoding helpers (FIELD_ORDERS like Cisco Duo)\nlocal FIELD_ORDERS = {\n root = {\n \"actor\", \"client\", \"device\", \"authenticationContext\", \"displayMessage\",\n \"eventType\", \"outcome\", \"published\", \"securityContext\", \"severity\",\n \"debugContext\", \"legacyEventType\", \"transaction\", \"uuid\", \"version\",\n \"request\", \"target\"\n },\n actor = {\"id\", \"type\", \"alternateId\", \"displayName\", \"detailEntry\"},\n client = {\"userAgent\", \"zone\", \"device\", \"id\", \"ipAddress\", \"geographicalContext\"},\n userAgent = {\"rawUserAgent\", \"os\", \"browser\"},\n geographicalContext = {\"city\", \"state\", \"country\", \"postalCode\", \"geolocation\"},\n geolocation = {\"lat\", \"lon\"},\n authenticationContext = {\n \"authenticationProvider\", \"credentialProvider\", \"credentialType\", \"issuer\",\n \"interface\", \"authenticationStep\", \"externalSessionId\"\n },\n issuer = {\"id\", \"type\"},\n outcome = {\"result\", \"reason\"},\n securityContext = {\"asNumber\", \"asOrg\", \"isp\", \"domain\", \"isProxy\"},\n debugContext = {\"debugData\"},\n debugData = {\"initiationType\", \"redirectUri\", \"requestId\", \"dtHash\", \"signOnMode\", \"requestUri\", \"threatSuspected\", \"url\", \"risk\", \"deviceFingerprint\", \"authnRequestId\", \"behaviors\", \"warningPercent\", \"rateLimitBucketUuid\", \"rateLimitSecondsToReset\", \"threshold\", \"timeSpan\", \"rateLimitScopeType\", \"userId\", \"timeUnit\"},\n transaction = {\"type\", \"id\", \"detail\"},\n request = {\"ipChain\"},\n ipChain = {\"ip\", \"geographicalContext\", \"version\", \"source\"},\n dst_endpoint = {\"uid\", \"svc_name\"},\n user = {\"id\", \"email_addr\", \"name\"},\n target = {\"id\", \"type\", \"alternateId\", \"displayName\", \"detailEntry\"}\n}\n\nlocal function encodeWithFieldOrder(obj, fieldOrder)\n local items = {}\n -- Phase 1: predefined order\n for _, fieldName in ipairs(fieldOrder) do\n if obj[fieldName] ~= nil then\n local valueStr = encodeJson(obj[fieldName], fieldName)\n if valueStr ~= nil then\n table.insert(items, '\"' .. fieldName .. '\": ' .. valueStr)\n end\n end\n end\n -- Phase 2: remaining fields\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then found = true; break end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local valueStr = encodeJson(v, k)\n if valueStr ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. valueStr)\n end\n end\n end\n -- If no items were added, return nil to skip empty objects\n if #items == 0 then\n return nil\n end\n return \"{\" .. table.concat(items, \", \") .. \"}\"\nend\n\n-- JSON encoding function (from AWS CloudTrail) - Compact single-line output\nfunction encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" or obj == \"\" then\n return nil -- Skip null and empty fields entirely\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n\n if isArray then\n local items = {}\n for i = 1, maxIndex do\n local elementKey = key or tostring(i)\n local encoded = encodeJson(obj[i], elementKey)\n if encoded ~= nil then\n table.insert(items, encoded)\n end\n end\n -- Return nil for empty arrays to skip them entirely\n if #items == 0 then\n return nil\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n -- Check if this is an empty object - if so, return nil to skip it\n if next(obj) == nil then\n return nil\n end\n\n -- If we have a field order for this object key, use it\n if key and FIELD_ORDERS[key] then\n return encodeWithFieldOrder(obj, FIELD_ORDERS[key])\n end\n -- If top-level message root, use root order\n if key == \"root\" and FIELD_ORDERS.root then\n return encodeWithFieldOrder(obj, FIELD_ORDERS.root)\n end\n -- Fallback: unordered (natural) encoding\n local items = {}\n for k, v in pairs(obj) do\n local keyStr = type(k) == \"string\" and k or tostring(k)\n local valueStr = encodeJson(v, keyStr)\n -- Only include fields that have non-nil values\n if valueStr ~= nil then\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. valueStr)\n end\n end\n -- If no items were added, return nil to skip empty objects\n if #items == 0 then\n return nil\n end\n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\n-- Get default mapping for event type\nfunction getDefaultMapping(eventType)\n local defaultMapping = {\n [\"user.session.start\"] = {\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"version\", target = \"api.version\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"authenticationContext.externalSessionId\", target = \"session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.authenticationProvider\", target = \"service.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.origin\", target = \"device.hostname\"},\n {source = \"debugContext.debugData.risk\", target = \"device.risk_level\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"}\n },\n [\"user.authentication.sso\"] = {\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"authenticationContext.externalSessionId\", target = \"session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.authenticationProvider\", target = \"service.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.signOnMode\", target = \"auth_protocol\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"},\n -- target fields\n {source = \"dst_endpoint_uid\", target = \"dst_endpoint.uid\"},\n {source = \"actor_user_uid\", target = \"actor.user.uid\"},\n {source = \"dst_endpoint_svc_name\", target = \"dst_endpoint.svc_name\"},\n {source = \"actor_email_addr\", target = \"actor.email_addr\"}\n --{source = \"actor_user_name\", target = \"user.name\"}\n },\n [\"user.lifecycle.activate\"] = {\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.browser\", target = \"client.userAgent.browser\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"}\n -- target\n --{source = \"user_id\", target = \"user.id\"}\n --{source = \"user_email_addr\", target = \"user.email_addr\"}\n --{source = \"user_name\", target = \"user.name\"}\n },\n [\"user.lifecycle.create\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestId\", target = \"api.request.uid\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"}\n -- target\n --{source = \"user_id\", target = \"user.id\"}\n --{source = \"user_email_addr\", target = \"user.email_addr\"},\n --{source = \"user_name\", target = \"user.name\"}\n },\n [\"user.lifecycle.deactivate\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.requestId\", target = \"api.request.uid\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"}\n -- target\n --{source = \"user_id\", target = \"user.id\"},\n --{source = \"user_email_addr\", target = \"user.email_addr\"},\n --{source = \"user_name\", target = \"user.name\"}\n },\n [\"policy.evaluate_sign_on\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"authenticationContext.authenticationProvider\", target = \"name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"authenticationContext.externalSessionId\", target = \"session.uid\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"debugContext.debugData.deviceFingerprint\", target = \"device.uid\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.risk\", target = \"device.risk_level\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"version\", target = \"api.version\"},\n -- target fields\n {source = \"actor.authorization.policy.uid\", target = \"actor.authorization.policy.uid\"},\n {source = \"actor.authorization.policy.name\", target = \"actor.authorization.policy.name\"},\n {source = \"actor.authorization.policy.rule.uid\", target = \"actor.authorization.policy.rule.uid\"},\n {source = \"actor.authorization.policy.rule.name\", target = \"actor.authorization.policy.rule.name\"}\n },\n [\"system.org.rate_limit.warning\"] = {\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"version\", target = \"api.version\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"}\n },\n [\"application.user_membership.add\"] = {\n {source = \"uuid\", target = \"metadata.uid\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"version\", target = \"api.version\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"client.ipAddress\", target = \"src_endpoint.ip\"},\n {source = \"client.id\", target = \"src_endpoint.uid\"},\n {source = \"client.geographicalContext.city\", target = \"src_endpoint.location.city\"},\n {source = \"client.geographicalContext.country\", target = \"src_endpoint.location.country\"},\n {source = \"client.geographicalContext.geolocation.lat\", target = \"location.coordinates.lat\"},\n {source = \"client.geographicalContext.geolocation.lon\", target = \"location.coordinates.lon\"},\n {source = \"client.geographicalContext.postalCode\", target = \"src_endpoint.location.postal_code\"},\n {source = \"client.geographicalContext.state\", target = \"src_endpoint.location.region\"},\n {source = \"client.userAgent.rawUserAgent\", target = \"http_request.user_agent\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"authenticationContext.externalSessionId\", target = \"actor.session.uid\"},\n {source = \"authenticationContext.issuer.id\", target = \"actor.idp.uid\"},\n {source = \"authenticationContext.issuer.type\", target = \"actor.idp.name\"},\n {source = \"authenticationContext.credentialProvider\", target = \"actor.invoked_by\"},\n {source = \"authenticationContext.credentialType\", target = \"actor.user.account.type\"},\n {source = \"debugContext.debugData.requestUri\", target = \"http_request.url.path\"},\n {source = \"debugContext.debugData.url\", target = \"http_request.url.query_string\"},\n {source = \"debugContext.debugData.requestId\", target = \"http_request.uid\"},\n {source = \"debugContext.debugData.origin\", target = \"device.hostname\"},\n {source = \"debugContext.debugData.risk\", target = \"device.risk_level\"},\n {source = \"securityContext.isp\", target = \"src_endpoint.location.isp\"},\n {source = \"securityContext.domain\", target = \"src_endpoint.location.domain\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n -- target fields\n {source = \"dst_endpoint_uid\", target = \"dst_endpoint.uid\"},\n {source = \"dst_endpoint_svc_name\", target = \"dst_endpoint.svc_name\"},\n {source = \"actor_user_uid\", target = \"actor.user.uid\"},\n {source = \"actor_email_addr\", target = \"actor.email_addr\"}\n --{source = \"actor_user_name\", target = \"user.name\"}\n },\n [\"application.lifecycle.update\"] = {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.type\", target = \"actor.user.type\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"},\n -- target fields\n {source = \"dst_endpoint_uid\", target = \"dst_endpoint.uid\"},\n {source = \"dst_endpoint_svc_name\", target = \"dst_endpoint.svc_name\"},\n\n --{source = \"actor_user_uid\", target = \"actor.user.uid\"},\n {source = \"actor_email_addr\", target = \"actor.email_addr\"}\n --{source = \"actor_user_name\", target = \"user.name\"}\n }\n }\n\n return defaultMapping[eventType] or getGenericOktaMapping()\nend\n\n-- Generic Okta mapping\nfunction getGenericOktaMapping()\n return {\n {source = \"actor.id\", target = \"actor.user.uid\"},\n {source = \"actor.alternateId\", target = \"actor.user.email_addr\"},\n {source = \"actor.displayName\", target = \"actor.user.name\"},\n {source = \"outcome.result\", target = \"status\"},\n {source = \"outcome.reason\", target = \"status_detail\"},\n {source = \"published\", target = \"metadata.original_time\"},\n {source = \"transaction.id\", target = \"metadata.correlation_uid\"},\n {source = \"transaction.detail\", target = \"raw_data\"},\n {source = \"uuid\", target = \"metadata.uid\"}\n }\nend\n\n-- Common mapping\nfunction getCommonMapping()\n return {\n {source = \"category_name\", target = \"category_name\"},\n {source = \"category_uid\", target = \"category_uid\"},\n {source = \"class_uid\", target = \"class_uid\"},\n {source = \"severity_id\", target = \"severity_id\"},\n {source = \"activity_name\", target = \"activity_name\"},\n {source = \"activity_id\", target = \"activity_id\"},\n {source = \"type_uid\", target = \"type_uid\"},\n {source = \"vendor_name\", target = \"metadata.product.vendor_name\"},\n {source = \"name\", target = \"metadata.product.name\"},\n {source = \"OCSF_version\", target = \"metadata.version\"},\n {source = \"time\", target = \"time\"},\n {source = \"status_id\", target = \"status_id\"},\n {source = \"type_id\", target = \"actor.user.type_id\"},\n {source = \"observables\", target = \"observables\"},\n {source = \"dataSource.category\", target = \"dataSource.category\"},\n {source = \"site.id\", target = \"site.id\"},\n {source = \"dataSource.name\", target = \"dataSource.name\"},\n {source = \"dataSource.vendor\", target = \"dataSource.vendor\"},\n {source = \"message\", target = \"message\"},\n {source = \"class_name\", target = \"class_name\"},\n {source = \"type_name\", target = \"type_name\"}\n }\nend\n\n-- Parse security domain\nfunction parseSecurityDomain(parsedLog)\n if parsedLog[\"src_endpoint.location.domain\"] == \".\" then\n parsedLog[\"src_endpoint.location.domain\"] = nil\n end\n return parsedLog\nend\n\n-- Set site ID\nfunction setSiteId(oktaLog, siteId)\n oktaLog[\"site\"] = {id = siteId}\n return oktaLog\nend\n\n-- Parse risk level\nfunction parseRiskLevel(parsedLog)\n local riskLevelMapper = {\n [\"Info\"] = 0, [\"Low\"] = 1, [\"Medium\"] = 2,\n [\"High\"] = 3, [\"Critical\"] = 4\n }\n local riskPrefix = \"level=\"\n local finalRiskLevel = \"Other\"\n local finalRiskLevelId = 99\n\n -- Look for risk level in the already-mapped device.risk_level field\n local riskLevel = \"\"\n if parsedLog[\"device\"] and parsedLog[\"device\"][\"risk_level\"] then\n riskLevel = parsedLog[\"device\"][\"risk_level\"]\n end\n \n for riskLevelName, riskLevelId in pairs(riskLevelMapper) do\n if string.find(string.lower(riskLevel), string.lower(riskPrefix .. riskLevelName)) then\n finalRiskLevel = riskLevelName\n finalRiskLevelId = riskLevelId\n break\n end\n end\n\n -- Set the parsed risk level back to the device object\n if not parsedLog[\"device\"] then parsedLog[\"device\"] = {} end\n parsedLog[\"device\"][\"risk_level\"] = finalRiskLevel\n parsedLog[\"device\"][\"risk_level_id\"] = finalRiskLevelId\n return parsedLog\nend\n\n-- Find event type\nfunction findEventType(log)\n local eventType = log[\"eventType\"]\n if eventType == nil then\n return \"unknown\"\n end\n return eventType\nend\n\n-- Get category UID\nfunction getCategoryUID(eventType)\n local categoryMapping = {\n [\"user.session.start\"] = 3,\n [\"user.authentication.sso\"] = 3,\n [\"user.lifecycle.activate\"] = 2,\n [\"user.lifecycle.create\"] = 3,\n [\"user.lifecycle.deactivate\"] = 3,\n [\"policy.evaluate_sign_on\"] = 3,\n [\"system.org.rate_limit.warning\"] = 3,\n [\"application.user_membership.add\"] = 3,\n [\"application.lifecycle.update\"] = 6\n }\n return categoryMapping[eventType] or 0\nend\n\n-- Get class mapping\nfunction getClassMapping(eventType)\n local classMapping = {\n [\"user.session.start\"] = {name = \"Authentication\", id = 3002},\n [\"user.authentication.sso\"] = {name = \"Authentication\", id = 3002},\n [\"user.lifecycle.activate\"] = {name = \"Account Change\", id = 3001},\n [\"user.lifecycle.create\"] = {name = \"Account Change\", id = 3001},\n [\"user.lifecycle.deactivate\"] = {name = \"Account Change\", id = 3001},\n [\"policy.evaluate_sign_on\"] = {name = \"Authentication\", id = 3002},\n [\"system.org.rate_limit.warning\"] = {name = \"API Activity\", id = 6003},\n [\"application.user_membership.add\"] = {name = \"Account Change\", id = 3001},\n [\"application.lifecycle.update\"] = {name = \"Application Lifecycle\", id = 6002}\n }\n local mapping = classMapping[eventType] or {name = \"Base Event\", id = 0}\n return mapping.name, mapping.id\nend\n\n-- Get status default OCSF mapping\nfunction getStatusDefaultOCSFMapping(status)\n if status == nil then\n return 0\n end\n\n local statusMapping = {\n [\"SUCCESS\"] = 1,\n [\"FAILURE\"] = 2\n }\n return statusMapping[status] or 99\nend\n\n-- Get activity name\nfunction getActivityName(eventType, eventTypeDisplayName)\n local activityMapping = {\n [\"user.session.start\"] = {name = \"Logon\", id = 1},\n [\"user.authentication.sso\"] = {name = \"Logon\", id = 1},\n [\"user.lifecycle.activate\"] = {name = \"Enable\", id = 2},\n [\"user.lifecycle.create\"] = {name = \"Create\", id = 1},\n [\"user.lifecycle.deactivate\"] = {name = \"Disable\", id = 5},\n [\"policy.evaluate_sign_on\"] = {name = \"Logon\", id = 1},\n [\"application.user_membership.add\"] = {name = \"Attach Policy\", id = 7}\n }\n local mapping = activityMapping[eventType] or {name = eventTypeDisplayName, id = 99}\n return mapping.name, mapping.id\nend\n\n-- Get user type\nfunction getUserType(oktaUserType)\n local userTypeMapping = {\n [\"Unknown\"] = 0,\n [\"User\"] = 1,\n [\"Admin\"] = 2,\n [\"System\"] = 3,\n [\"Other\"] = 99\n }\n for user, id in pairs(userTypeMapping) do\n if string.find(oktaUserType or \"\", user) then\n return id\n end\n end\n return 99\nend\n\n-- Get category mapper\nfunction getCategoryMapper(eventType)\n local categoryMapper = {\n [\"user.session.start\"] = \"Identity & Access Management\",\n [\"user.authentication.sso\"] = \"Identity & Access Management\",\n [\"user.lifecycle.activate\"] = \"Identity & Access Management\",\n [\"user.lifecycle.create\"] = \"Identity & Access Management\",\n [\"user.lifecycle.deactivate\"] = \"Identity & Access Management\",\n [\"policy.evaluate_sign_on\"] = \"Identity & Access Management\",\n [\"system.org.rate_limit.warning\"] = \"Application Activity\",\n [\"application.user_membership.add\"] = \"Identity & Access Management\",\n [\"application.lifecycle.update\"] = \"Application Activity\"\n }\n return categoryMapper[eventType] or \"Uncategorized\"\nend\n\n-- Get type name\nfunction getTypeName(eventType)\n local typeMapper = {\n [\"user.session.start\"] = \"Authentication: Logon\",\n [\"user.authentication.sso\"] = \"Authentication: Logon\",\n [\"user.lifecycle.activate\"] = \"Account Change: Enable\",\n [\"user.lifecycle.create\"] = \"Account Change: Create\",\n [\"user.lifecycle.deactivate\"] = \"Account Change: Disable\",\n [\"policy.evaluate_sign_on\"] = \"Authentication: Logon\",\n [\"system.org.rate_limit.warning\"] = \"API Activity: Other\",\n [\"application.user_membership.add\"] = \"Account Change: Attach Policy\",\n [\"application.lifecycle.update\"] = \"Application Lifecycle: Other\"\n }\n return typeMapper[eventType] or \"Base Event: Other\"\nend\n\n-- Get observables\nfunction getObservables(log)\n local observables = {}\n\n -- Hostname observable\n local hostname = safelyAccessNestedDictKeys({\"debugContext\", \"debugData\", \"origin\"}, log)\n if hostname and hostname ~= \"\" and hostname ~= \"null\" then\n table.insert(observables, {\n type_id = 1,\n type = \"Hostname\",\n name = \"device.hostname\",\n value = hostname\n })\n end\n\n -- IP Address observable\n local clientIpAddress = safelyAccessNestedDictKeys({\"client\", \"ipAddress\"}, log)\n if clientIpAddress and clientIpAddress ~= \"\" and clientIpAddress ~= \"null\" then\n table.insert(observables, {\n type_id = 2,\n type = \"IP Address\",\n name = \"src_endpoint.ip\",\n value = clientIpAddress\n })\n end\n\n -- User Name observable\n local userName = safelyAccessNestedDictKeys({\"actor\", \"displayName\"}, log)\n if userName and userName ~= \"\" and userName ~= \"null\" then\n table.insert(observables, {\n type_id = 4,\n type = \"User Name\",\n name = \"actor.user.name\",\n value = userName\n })\n end\n\n -- Email Address observable\n local emailAddress = safelyAccessNestedDictKeys({\"actor\", \"alternateId\"}, log)\n if emailAddress and emailAddress ~= \"\" and emailAddress ~= \"null\" then\n table.insert(observables, {\n type_id = 5,\n type = \"Email Address\",\n name = \"actor.user.email_addr\",\n value = emailAddress\n })\n end\n\n -- URL String observable\n local requestUri = safelyAccessNestedDictKeys({\"debugContext\", \"debugData\", \"requestUri\"}, log)\n if requestUri and requestUri ~= \"\" and requestUri ~= \"null\" then\n table.insert(observables, {\n type_id = 6,\n type = \"URL String\",\n name = \"http_request.url.path\",\n value = requestUri\n })\n end\n\n -- Geo Location observable\n local lat = safelyAccessNestedDictKeys({\"client\", \"geographicalContext\", \"geolocation\", \"lat\"}, log)\n local lon = safelyAccessNestedDictKeys({\"client\", \"geographicalContext\", \"geolocation\", \"lon\"}, log)\n local notAllowedItems = {{}, {}, \"\", nil}\n local latAllowed = true\n local lonAllowed = true\n for _, item in ipairs(notAllowedItems) do\n if lat == item then latAllowed = false end\n if lon == item then lonAllowed = false end\n end\n if latAllowed and lonAllowed and lat and lon then\n table.insert(observables, {\n type_id = 26,\n type = \"Geo Location\",\n name = \"client.geographicalContext.geolocation\",\n value = lat .. \", \" .. lon\n })\n end\n\n return observables\nend\n\n-- Process target fields into flat keys for mapping\nfunction processTargetFields(log, eventType)\n local targetFields = {}\n \n if eventType == \"user.authentication.sso\" or\n eventType == \"application.user_membership.add\" or\n eventType == \"application.lifecycle.update\" then\n local target = log[\"target\"]\n if target and type(target) == \"table\" then\n for _, t in ipairs(target) do\n if type(t) == \"table\" then\n if t[\"type\"] == \"AppInstance\" then\n log[\"dst_endpoint_uid\"] = t[\"type\"]\n log[\"dst_endpoint_svc_name\"] = t[\"displayName\"]\n elseif t[\"type\"] == \"AppUser\" then\n -- Override specific fields with target values\n log[\"actor_user_uid\"] = t[\"type\"] -- \"AppUser\" overrides actor.id\n log[\"actor_email_addr\"] = t[\"alternateId\"] -- \"unknown\" for actor.email_addr\n log[\"actor_user_name\"] = t[\"displayName\"] -- \"John Cena\" (same as original)\n \n -- Override actor.id for actor.user.uid mapping\n if log[\"actor\"] then\n log[\"actor\"][\"id\"] = t[\"type\"] -- \"AppUser\" for actor.user.uid\n -- Keep actor.alternateId as original for actor.user.email_addr mapping\n -- Don't override actor.alternateId - let it stay as \"john.cena@wwe.com\"\n end\n end\n end\n end\n end\n elseif eventType == \"user.lifecycle.activate\" or\n eventType == \"user.lifecycle.create\" or\n eventType == \"user.lifecycle.deactivate\" then\n local target = log[\"target\"]\n if target and type(target) == \"table\" and #target > 0 then\n local t = target[1]\n if type(t) == \"table\" then\n targetFields[\"user_id\"] = t[\"id\"]\n targetFields[\"user_email_addr\"] = t[\"alternateId\"]\n targetFields[\"user_name\"] = t[\"displayName\"]\n end\n end\n elseif eventType == \"policy.evaluate_sign_on\" then\n local target = log[\"target\"]\n if target and type(target) == \"table\" then\n for _, t in ipairs(target) do\n if type(t) == \"table\" then\n if t[\"type\"] == \"PolicyEntity\" then\n targetFields[\"actor\"] = {\n authorization = {\n policy = {\n name = t[\"displayName\"],\n uid = t[\"id\"]\n }\n }\n }\n elseif t[\"type\"] == \"PolicyRule\" then\n targetFields[\"actor\"] = {\n authorization = {\n policy = {\n rule = {\n uid = t[\"id\"],\n name = t[\"displayName\"]\n }\n }\n }\n }\n end\n end\n end\n end\n end\n\n return targetFields\nend\n\n-- Generate severity mapping\nfunction generateSeverityMapping(availableSeverityList)\n local defaultSeverityMapping = {\n [\"DEBUG\"] = 0, [\"INFO\"] = 1, [\"WARN\"] = 3, [\"ERROR\"] = 5, [\"OTHER\"] = 99\n }\n local defaultSeverityMappingKeys = {\"DEBUG\", \"INFO\", \"WARN\", \"ERROR\", \"OTHER\"}\n local severityIDMapping = {}\n\n for severityTypeIndex = 1, #availableSeverityList do\n if availableSeverityList[severityTypeIndex] then\n local key = defaultSeverityMappingKeys[severityTypeIndex]\n severityIDMapping[key] = defaultSeverityMapping[key]\n end\n end\n return severityIDMapping\nend\n\n-- Get severity ID\nfunction getSeverityID(eventType, logSeverity)\n local severityMapping = {\n [\"user.session.start\"] = generateSeverityMapping({true, true, true, true, true}),\n [\"user.authentication.sso\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"user.lifecycle.activate\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"user.lifecycle.create\"] = generateSeverityMapping({false, true, false, false, false}),\n [\"user.lifecycle.deactivate\"] = generateSeverityMapping({false, true, false, false, false}),\n [\"policy.evaluate_sign_on\"] = generateSeverityMapping({false, true, false, false, false}),\n [\"system.org.rate_limit.warning\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"application.user_membership.add\"] = generateSeverityMapping({true, true, true, false, false}),\n [\"application.lifecycle.update\"] = generateSeverityMapping({false, true, false, false, false})\n }\n local eventSeverityMapping = severityMapping[eventType] or {}\n return eventSeverityMapping[logSeverity] or 99\nend\n\n-- Generate synthetic fields\nfunction generateSyntheticFields(log, eventType, originalLog)\n -- Set nested metadata fields\n if not log[\"metadata\"] then log[\"metadata\"] = {} end\n if not log[\"metadata\"][\"product\"] then log[\"metadata\"][\"product\"] = {} end\n log[\"metadata\"][\"product\"][\"vendor_name\"] = \"Okta\"\n log[\"metadata\"][\"product\"][\"name\"] = \"Okta\"\n log[\"metadata\"][\"version\"] = \"1.0.0\"\n\n local publishedTime = safelyAccessNestedDictKeys({\"published\"}, originalLog)\n if publishedTime then\n log[\"time\"] = convertToMilliseconds(publishedTime)\n end\n\n -- Use the event type passed as parameter instead of finding it again\n log[\"category_name\"] = getCategoryMapper(eventType)\n log[\"category_uid\"] = getCategoryUID(eventType)\n\n local className, classUid = getClassMapping(eventType)\n log[\"class_name\"] = className\n log[\"class_uid\"] = classUid\n\n -- Dynamic severity calculation\n log[\"severity_id\"] = getSeverityID(eventType, originalLog[\"severity\"])\n\n local activityName, activityId = getActivityName(eventType, originalLog[\"displayMessage\"] or \"Other\")\n log[\"activity_name\"] = activityName\n log[\"activity_id\"] = activityId\n\n log[\"type_uid\"] = (classUid * 100) + activityId\n log[\"type_name\"] = getTypeName(eventType)\n\n local outcomeResult = safelyAccessNestedDictKeys({\"outcome\", \"result\"}, originalLog)\n log[\"status_id\"] = getStatusDefaultOCSFMapping(outcomeResult)\n\n local actorType = safelyAccessNestedDictKeys({\"actor\", \"type\"}, originalLog)\n\n\tif not log[\"actor\"] then log[\"actor\"] = {} end\n\tif not log[\"actor\"][\"user\"] then log[\"actor\"][\"user\"] = {} end\n\tlog[\"actor\"][\"user\"][\"type_id\"] = getUserType(actorType)\n\tlog[\"actor\"][\"user\"][\"type\"] = actorType or \"User\"\n -- Actor email_addr is now set by target field mappings, don't override\n\n -- Handle postal code conversion to string\n local postalCode = safelyAccessNestedDictKeys({\"client\", \"geographicalContext\", \"postalCode\"}, log)\n if postalCode then\n if not log[\"client\"] then log[\"client\"] = {} end\n if not log[\"client\"][\"geographicalContext\"] then log[\"client\"][\"geographicalContext\"] = {} end\n log[\"client\"][\"geographicalContext\"][\"postalCode\"] = tostring(postalCode)\n end\n\n -- Target fields are now processed earlier in oktaLogsMapping\n\n\n log[\"event.type\"] = log[\"activity_name\"] or eventType\n log[\"dataSource\"] = {name = \"Okta\", category = \"security\", vendor = \"Okta\"}\n \n -- Add session field (skip root session for lifecycle events except deactivate)\n local sessionId = safelyAccessNestedDictKeys({\"authenticationContext\", \"externalSessionId\"}, originalLog)\n if not (eventType == \"user.lifecycle.activate\" or eventType == \"user.lifecycle.create\") then\n if eventType == \"user.lifecycle.deactivate\" then\n -- For deactivate events, add session to existing actor object if it exists\n if log[\"actor\"] then\n log[\"actor\"][\"session\"] = {uid = sessionId or \"unknown\"}\n else\n log[\"session\"] = {uid = sessionId or \"unknown\"}\n end\n end\n end\n \n -- Add user field from actor only for non-lifecycle events\n --local userName = safelyAccessNestedDictKeys({\"actor\", \"displayName\"}, originalLog)\n --if userName and not (eventType == \"user.lifecycle.activate\" or eventType == \"user.lifecycle.create\" or eventType == \"user.lifecycle.deactivate\" or eventType == \"user.session.start\") then\n -- log[\"user\"] = {name = userName}\n --end\n\n return log\nend\n\n\n-- Helper function to check if a field should be ignored\nfunction shouldIgnoreField(fieldName)\n\n local ignoreFields = {\n \"_okta_event_type\", \"_ob\", \"ts\", \"timestamp\"\n }\n\n for _, field in ipairs(ignoreFields) do\n if fieldName == field or string.find(fieldName, \"^\" .. field .. \"%.\") then\n return true\n end\n end\n return false\nend\n\n-- Helper function to check if a value is empty (empty object, array, or null)\nfunction isEmptyValue(value)\n if value == nil or value == \"NULL_PLACEHOLDER\" then\n return true\n end\n if type(value) == \"string\" and (value == \"\" or value == \"null\") then\n return true\n end\n if type(value) == \"table\" then\n -- Check if it's an empty table\n if next(value) == nil then\n return true\n end\n -- For unmapped fields, recursively check nested values to filter out empty nested objects\n local hasNonEmptyValues = false\n for k, v in pairs(value) do\n if not isEmptyValue(v) then\n hasNonEmptyValues = true\n break\n end\n end\n return not hasNonEmptyValues\n end\n return false\nend\n\n-- Helper function to add unmapped fields as a truly nested object (no dotted keys)\nfunction addUnmappedFields(sourceObj, targetObj, mappedFields, prefixParts)\n prefixParts = prefixParts or {}\n for key, value in pairs(sourceObj) do\n -- Skip ignored fields\n if shouldIgnoreField(key) then\n goto continue\n end\n\n -- Build current dotted path for mapped check\n local currentPathParts = {}\n for i = 1, #prefixParts do currentPathParts[i] = prefixParts[i] end\n table.insert(currentPathParts, key)\n local currentPath = table.concat(currentPathParts, \".\")\n\n -- Check if this exact path has been mapped\n local isMapped = mappedFields[currentPath] or false\n\n if not isMapped and value ~= nil then\n if type(value) == \"table\" then\n local nestedObj = {}\n addUnmappedFields(value, nestedObj, mappedFields, currentPathParts)\n \n -- Check if ALL direct children were mapped\n local allChildrenMapped = true\n for childKey, childValue in pairs(value) do\n local childPath = currentPath .. \".\" .. childKey\n if not mappedFields[childPath] then\n allChildrenMapped = false\n break\n end\n end\n \n -- Only add parent if it has unmapped children OR no children were mapped\n if next(nestedObj) and not allChildrenMapped then\n targetObj[key] = nestedObj\n end\n -- Don't add empty objects to unmapped\n else\n -- Only add non-empty values to unmapped\n if not isEmptyValue(value) then\n targetObj[key] = value\n end\n end\n end\n ::continue::\n end\nend\n\n-- Helper function to build nested object structure from flat dotted keys\nfunction buildNestedStructure(flatObj)\n local nested = {}\n for key, value in pairs(flatObj) do\n local keys = split(key, \".\")\n local current = nested\n for i = 1, #keys - 1 do\n local k = keys[i]\n if not current[k] then\n current[k] = {}\n elseif type(current[k]) ~= \"table\" then\n -- If the existing value is not a table, we can't create nested structure\n -- Skip this key to avoid conflicts\n goto continue\n end\n current = current[k]\n end\n \n -- Special handling for raw_data field - encode as JSON string\n if keys[#keys] == \"raw_data\" then\n -- For raw_data, we want the value as a JSON string, not double-encoded\n if type(value) == \"table\" then\n current[keys[#keys]] = encodeJson(value, \"raw_data\")\n else\n -- If it's already a string, use it as-is\n current[keys[#keys]] = tostring(value)\n end\n else\n current[keys[#keys]] = value\n end\n ::continue::\n end\n return nested\nend\n\n-- Helper function to set nested value (marks field as processed)\nfunction setNestedValue(obj, keys, value)\n local current = obj\n for i = 1, #keys - 1 do\n if not current[keys[i]] then\n current[keys[i]] = {}\n end\n current = current[keys[i]]\n end\n current[keys[#keys]] = value\nend\n\n\n-- Helper function to filter out ignored fields using shouldIgnoreField\nfunction filterIgnoredFields(log)\n local function filterObject(obj, prefix)\n prefix = prefix or \"\"\n local filteredObj = {}\n\n for key, value in pairs(obj) do\n local fullKey = prefix == \"\" and key or prefix .. \".\" .. key\n\n if not shouldIgnoreField(fullKey) then\n if type(value) == \"table\" then\n local filteredValue = filterObject(value, fullKey)\n if next(filteredValue) then -- Only add non-empty tables\n filteredObj[key] = filteredValue\n end\n else\n filteredObj[key] = value\n end\n end\n end\n return filteredObj\n end\n\n return filterObject(log)\nend\n\n\n-- Helper function to create ordered JSON message using FIELD_ORDER approach\nfunction createOrderedMessage(log, eventType)\n -- Filter out null values and empty strings from the log before creating message\n local filteredLog = {}\n for k, v in pairs(log) do\n if v ~= nil and v ~= \"\" and v ~= \"null\" then\n if type(v) == \"table\" then\n local filteredValue = filterNullValues(v)\n if next(filteredValue) then -- Only add non-empty tables\n filteredLog[k] = filteredValue\n end\n else\n filteredLog[k] = v\n end\n end\n end\n \n -- Apply domain filtering to the message for lifecycle events only\n if eventType == \"user.lifecycle.create\" or eventType == \"user.lifecycle.deactivate\" or eventType == \"user.session.start\" then\n filteredLog = applyDomainFiltering(filteredLog)\n end\n \n -- Use FIELD_ORDERS.root to create ordered JSON string\n local orderedJson = encodeWithFieldOrder(filteredLog, FIELD_ORDERS.root)\n return orderedJson or \"{}\"\nend\n\n-- Helper function to recursively filter out null values from nested tables\nfunction filterNullValues(obj)\n local filtered = {}\n for k, v in pairs(obj) do\n if v ~= nil and v ~= \"\" and v ~= \"null\" then\n if type(v) == \"table\" then\n local filteredValue = filterNullValues(v)\n if next(filteredValue) then -- Only add non-empty tables\n filtered[k] = filteredValue\n end\n else\n filtered[k] = v\n end\n end\n end\n return filtered\nend\n\n-- Helper function to apply domain filtering recursively\nfunction applyDomainFiltering(obj)\n if type(obj) ~= \"table\" then\n return obj\n end\n \n local filtered = {}\n for k, v in pairs(obj) do\n if type(v) == \"table\" then\n local filteredValue = applyDomainFiltering(v)\n if next(filteredValue) then -- Only add non-empty tables\n filtered[k] = filteredValue\n end\n else\n -- Skip domain fields with value \".\" and isp fields that duplicate asOrg\n if k == \"domain\" and v == \".\" then\n -- Skip this field\n elseif k == \"isp\" and obj[\"asOrg\"] and v == obj[\"asOrg\"] then\n -- Skip isp field when it's the same as asOrg\n else\n filtered[k] = v\n end\n end\n end\n return filtered\nend\n\n-- Simplified Okta logs mapping function\nfunction oktaLogsMapping(log)\n -- STEP 1: First filter out ignored fields using shouldIgnoreField\n log = filterIgnoredFields(log)\n\n -- STEP 2: Find event type first\n local eventType = findEventType(log)\n \n -- STEP 3: Create ordered JSON message from original log BEFORE any modifications\n local originalLog = {}\n for k, v in pairs(log) do\n originalLog[k] = v\n end\n log.message = createOrderedMessage(originalLog, eventType)\n\n -- STEP 4: Process target fields to create flat keys for mapping\n local targetFields = processTargetFields(log, eventType)\n \n -- Apply target fields to log for specific events so mappings can consume them\n if eventType == \"policy.evaluate_sign_on\" then\n for key, value in pairs(targetFields) do\n if key == \"actor\" then\n if not log[\"actor\"] then log[\"actor\"] = {} end\n for subKey, subValue in pairs(value) do\n log[\"actor\"][subKey] = subValue\n end\n else\n log[key] = value\n end\n end\n elseif eventType == \"user.lifecycle.activate\" or eventType == \"user.lifecycle.create\" or eventType == \"user.lifecycle.deactivate\" then\n -- For lifecycle events, expose target-derived user fields on the log\n for key, value in pairs(targetFields) do\n log[key] = value\n end\n end\n\n -- STEP 4: Get mapping for event type\n local mappings = getDefaultMapping(eventType)\n\n -- Merge with common mapping\n local commonMappings = getCommonMapping()\n for _, mapping in ipairs(commonMappings) do\n table.insert(mappings, mapping)\n end\n\n -- STEP 4: Apply mappings and track processed fields\n local parsedData = {}\n local mappedFields = {}\n for _, mapping in ipairs(mappings) do\n local sourcePath = mapping.source\n local targetPath = mapping.target\n local keys = split(sourcePath, \".\")\n local value = safelyAccessNestedDictKeys(keys, log)\n\n if value ~= nil and value ~= \"\" and value ~= \"null\" then\n if targetPath == \"src_endpoint.location.domain\" and value == \".\" then\n -- Skip this field for any event when domain is a single dot\n else\n -- Update target in parsedData\n parsedData[targetPath] = value\n -- Track mapped field by source path\n mappedFields[sourcePath] = true\n end\n end\n end\n\n -- STEP 5: Add remaining non-null fields to unmapped\n parsedData[\"unmapped\"] = {}\n local ignoreUnmappedFields = {\n \"actor_user_uid\",\n \"actor_email_addr\", \n \"actor_user_name\",\n \"user_id\",\n \"user_email_addr\",\n \"user_name\"\n }\n for _, field in ipairs(ignoreUnmappedFields) do\n mappedFields[field] = true\n end\n\n addUnmappedFields(log, parsedData[\"unmapped\"], mappedFields, {})\n \n -- Apply domain filtering to unmapped fields as well\n parsedData[\"unmapped\"] = applyDomainFiltering(parsedData[\"unmapped\"])\n\n -- STEP 6: Convert flat dotted keys to nested objects\n parsedData = buildNestedStructure(parsedData)\n\n -- STEP 7: Apply post-processing\n parsedData = parseSecurityDomain(parsedData)\n parsedData = parseRiskLevel(parsedData)\n\n -- STEP 8: Generate synthetic fields\n parsedData = generateSyntheticFields(parsedData, eventType, log)\n \n -- STEP 9: Generate observables for specific event types (after synthetic fields)\n local observablesEventTypes = {\n \"user.authentication.sso\",\n \"application.user_membership.add\",\n \"user.lifecycle.activate\",\n \"user.lifecycle.create\",\n \"user.lifecycle.deactivate\",\n \"policy.evaluate_sign_on\",\n \"system.org.rate_limit.warning\",\n \"user.session.start\"\n }\n\n for _, eventTypeName in ipairs(observablesEventTypes) do\n if eventType == eventTypeName then\n local observables = getObservables(log)\n -- Convert observables to array format\n local observablesArray = {}\n for _, obs in ipairs(observables) do\n table.insert(observablesArray, obs)\n end\n parsedData[\"observables\"] = observablesArray\n break\n end\n end\n \n parsedData = convertToNested(parsedData, eventType)\n return parsedData\nend\n\n-- Convert flat parsed data to nested JSON structure with field ordering\nfunction convertToNested(parsedData, eventType)\n local nested = {}\n \n -- First, build the nested structure\n for key, value in pairs(parsedData) do\n -- Special case: keep event.type as flattened field (don't nest it)\n if key == \"event.type\" then\n nested[\"event.type\"] = value -- Keep as flattened event.type\n goto continue\n end\n \n local keys = split(key, \".\")\n local current = nested\n \n -- Navigate to the parent object\n for i = 1, #keys - 1 do\n local k = keys[i]\n if not current[k] then\n current[k] = {}\n elseif type(current[k]) ~= \"table\" then\n -- If the existing value is not a table, we can't create nested structure\n -- Skip this key to avoid conflicts\n goto continue\n end\n current = current[k]\n end\n \n -- Set the final value - special handling for raw_data field\n if keys[#keys] == \"raw_data\" then\n -- For raw_data, we want the value as a JSON string, not double-encoded\n if type(value) == \"table\" then\n current[keys[#keys]] = encodeJson(value, \"raw_data\")\n else\n -- If it's already a string, use it as-is\n current[keys[#keys]] = tostring(value)\n end\n else\n current[keys[#keys]] = value\n end\n ::continue::\n end\n \n -- Then, apply field ordering to the nested structure\n return applyFieldOrdering(nested, nil, eventType)\nend\n\n-- Apply field ordering to nested structure\nfunction applyFieldOrdering(obj, fieldOrder, eventType)\n fieldOrder = fieldOrder or FIELD_ORDERS.root\n local ordered = {}\n \n -- Phase 1: Add fields in predefined order\n for _, fieldName in ipairs(fieldOrder) do\n if obj[fieldName] ~= nil then\n if type(obj[fieldName]) == \"table\" then\n -- Recursively apply ordering to nested objects\n local nestedFieldOrder = FIELD_ORDERS[fieldName]\n ordered[fieldName] = applyFieldOrdering(obj[fieldName], nestedFieldOrder, eventType)\n else\n ordered[fieldName] = obj[fieldName]\n end\n end\n end\n \n -- Phase 2: Add remaining fields not in the predefined order\n for key, value in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if key == fieldName then \n found = true\n break \n end\n end\n \n if not found then\n if type(value) == \"table\" then\n -- Recursively apply ordering to nested objects\n local nestedFieldOrder = FIELD_ORDERS[key]\n ordered[key] = applyFieldOrdering(value, nestedFieldOrder, eventType)\n else\n ordered[key] = value\n end\n end\n end\n \n return ordered\nend\n\n-- Convert ISO 8601 timestamp to Unix epoch milliseconds\nfunction convertToMilliseconds(timestamp)\n if not timestamp or timestamp == \"\" then\n return nil\n end\n \n -- Parse ISO 8601 format: \"2025-09-29T09:15:40Z\" or \"2025-09-29T09:15:40.123Z\"\n local year, month, day, hour, min, sec, ms = string.match(timestamp, \"(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z\")\n \n if year and month and day and hour and min and sec then\n local t = {\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec),\n isdst = false\n }\n \n -- Get local time interpretation\n local local_seconds = os.time(t)\n \n -- Get what this represents in UTC\n local utc_date = os.date(\"!*t\", local_seconds)\n \n local unix_seconds\n -- Check if the UTC interpretation matches our input\n if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and \n utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and \n utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then\n -- We are already in UTC, use as-is\n unix_seconds = local_seconds\n else\n -- Calculate the correct UTC timestamp\n local utc_seconds = os.time(utc_date)\n local offset = local_seconds - utc_seconds\n unix_seconds = local_seconds + offset -- Add offset to get UTC\n end\n \n -- Add milliseconds if present\n local milli = 0\n if ms and ms ~= \"\" then\n milli = tonumber((ms .. \"000\"):sub(1, 3)) -- pad/truncate to 3 digits\n end\n \n return unix_seconds * 1000 + milli\n end\n \n return nil\nend\n\n-- Main event processing function\nfunction processEvent(event)\n return oktaLogsMapping(event)\nend", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "F", - "harness_score": 45, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/okta/sample.json b/pipelines/community/transform_ocsf/okta/sample.json deleted file mode 100644 index 31471e0..0000000 --- a/pipelines/community/transform_ocsf/okta/sample.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "actor": { - "id": "00u4729hjsVRU197Y5d7", - "type": "User", - "alternateId": "pnnpydhb@example.com", - "displayName": "Pnnpydhb", - "detailEntry": null - }, - "client": { - "userAgent": { - "rawUserAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "os": "Linux", - "browser": "CHROME" - }, - "zone": "null", - "device": "Computer", - "id": null, - "ipAddress": "95.196.9.145", - "geographicalContext": { - "city": "Hyderabad", - "state": "Telangana", - "country": "India", - "postalCode": "500004", - "geolocation": { - "lat": 17.3724, - "lon": 78.4378 - } - } - }, - "device": null, - "authenticationContext": { - "authenticationProvider": null, - "credentialProvider": null, - "credentialType": null, - "issuer": null, - "interface": null, - "authenticationStep": 0, - "rootSessionId": "102eA1R", - "externalSessionId": "102eA1R2M" - }, - "displayMessage": "Create API token", - "eventType": "system.api_token.create", - "outcome": { - "result": "SUCCESS", - "reason": null - }, - "published": "2026-04-20T02:26:37.000Z", - "securityContext": { - "asNumber": 150008, - "asOrg": "s r fibernet", - "isp": "pioneer elabs ltd.", - "domain": null, - "isProxy": false - }, - "severity": "OTHER", - "debugContext": { - "debugData": { - "concurrencyPercentage": "50", - "requestId": "923c54e8-a3b4-4518-bc64-00b228adbe84", - "dtHash": "43fa03767e62cd45bf1b9ecf03ca2ee6f8d42c1248d6c3d5adcf990c055128a0", - "rateLimitPercentage": "50", - "networkConnection": "ANYWHERE", - "requestUri": "/api/internal/tokens", - "url": "/api/internal/tokens?expand=user" - } - }, - "legacyEventType": "api.token.create", - "transaction": { - "type": "WEB", - "id": "bc6400b228adbe84923c54e8a3b44518", - "detail": {} - }, - "uuid": "b2ece00c-6728-4e85-a08b-b0e3ccdcbaf2", - "version": "0", - "request": { - "ipChain": [ - { - "ip": "103.99.111.142", - "geographicalContext": { - "city": "Hyderabad", - "state": "Telangana", - "country": "India", - "postalCode": "500004", - "geolocation": { - "lat": 17.3724, - "lon": 78.4378 - } - }, - "version": "V4", - "source": null - } - ] - }, - "target": [ - { - "id": "00T2lyn19", - "type": "Token", - "alternateId": "unknown", - "displayName": "Test", - "detailEntry": null - } - ] -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/okta/serializer.lua b/pipelines/community/transform_ocsf/okta/serializer.lua deleted file mode 100644 index c3897c8..0000000 --- a/pipelines/community/transform_ocsf/okta/serializer.lua +++ /dev/null @@ -1,1495 +0,0 @@ - --- Lua implementation for Okta OCSF 1.0.0 schema - --- Helper function to safely access nested dictionary keys -function safelyAccessNestedDictKeys(keys, dictObject) - local current = dictObject - for _, key in ipairs(keys) do - if current and type(current) == "table" then - current = current[key] - else - return nil - end - end - return current -end - --- Helper function to split string by delimiter -function split(str, delimiter) - local result = {} - -- Escape special regex characters in delimiter - local escapedDelimiter = delimiter:gsub("[%.%+%*%?%^%$%(%)%[%]%%]", "%%%1") - local pattern = "([^" .. escapedDelimiter .. "]+)" - - for token in str:gmatch(pattern) do - table.insert(result, token) - end - - -- Handle empty string case - if #result == 0 and #str > 0 then - table.insert(result, str) - end - - return result -end - --- Helper function to convert timestamp to milliseconds -function convertToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - - -- Parse ISO 8601 timestamp (e.g., "2023-04-24T04:55:30.535Z") - local year, month, day, hour, min, sec, ms = timestamp:match("(%d%d%d%d)-(%d%d)-(%d%d)T(%d%d):(%d%d):(%d%d)%.(%d%d%d)Z") - - if year and month and day and hour and min and sec and ms then - -- Convert to milliseconds since epoch - local time = os.time({ - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec) - }) - return (time * 1000) + tonumber(ms) - end - - return nil -end - --- Ordered JSON encoding helpers (FIELD_ORDERS like Cisco Duo) -local FIELD_ORDERS = { - root = { - "actor", "client", "device", "authenticationContext", "displayMessage", - "eventType", "outcome", "published", "securityContext", "severity", - "debugContext", "legacyEventType", "transaction", "uuid", "version", - "request", "target" - }, - actor = {"id", "type", "alternateId", "displayName", "detailEntry"}, - client = {"userAgent", "zone", "device", "id", "ipAddress", "geographicalContext"}, - userAgent = {"rawUserAgent", "os", "browser"}, - geographicalContext = {"city", "state", "country", "postalCode", "geolocation"}, - geolocation = {"lat", "lon"}, - authenticationContext = { - "authenticationProvider", "credentialProvider", "credentialType", "issuer", - "interface", "authenticationStep", "externalSessionId" - }, - issuer = {"id", "type"}, - outcome = {"result", "reason"}, - securityContext = {"asNumber", "asOrg", "isp", "domain", "isProxy"}, - debugContext = {"debugData"}, - debugData = {"initiationType", "redirectUri", "requestId", "dtHash", "signOnMode", "requestUri", "threatSuspected", "url", "risk", "deviceFingerprint", "authnRequestId", "behaviors", "warningPercent", "rateLimitBucketUuid", "rateLimitSecondsToReset", "threshold", "timeSpan", "rateLimitScopeType", "userId", "timeUnit"}, - transaction = {"type", "id", "detail"}, - request = {"ipChain"}, - ipChain = {"ip", "geographicalContext", "version", "source"}, - dst_endpoint = {"uid", "svc_name"}, - user = {"id", "email_addr", "name"}, - target = {"id", "type", "alternateId", "displayName", "detailEntry"} -} - -local function encodeWithFieldOrder(obj, fieldOrder) - local items = {} - -- Phase 1: predefined order - for _, fieldName in ipairs(fieldOrder) do - if obj[fieldName] ~= nil then - local valueStr = encodeJson(obj[fieldName], fieldName) - if valueStr ~= nil then - table.insert(items, '"' .. fieldName .. '": ' .. valueStr) - end - end - end - -- Phase 2: remaining fields - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then found = true; break end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - local valueStr = encodeJson(v, k) - if valueStr ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. valueStr) - end - end - end - -- If no items were added, return nil to skip empty objects - if #items == 0 then - return nil - end - return "{" .. table.concat(items, ", ") .. "}" -end - --- JSON encoding function (from AWS CloudTrail) - Compact single-line output -function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" or obj == "" then - return nil -- Skip null and empty fields entirely - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray then - local items = {} - for i = 1, maxIndex do - local elementKey = key or tostring(i) - local encoded = encodeJson(obj[i], elementKey) - if encoded ~= nil then - table.insert(items, encoded) - end - end - -- Return nil for empty arrays to skip them entirely - if #items == 0 then - return nil - end - return "[" .. table.concat(items, ", ") .. "]" - else - -- Check if this is an empty object - if so, return nil to skip it - if next(obj) == nil then - return nil - end - - -- If we have a field order for this object key, use it - if key and FIELD_ORDERS[key] then - return encodeWithFieldOrder(obj, FIELD_ORDERS[key]) - end - -- If top-level message root, use root order - if key == "root" and FIELD_ORDERS.root then - return encodeWithFieldOrder(obj, FIELD_ORDERS.root) - end - -- Fallback: unordered (natural) encoding - local items = {} - for k, v in pairs(obj) do - local keyStr = type(k) == "string" and k or tostring(k) - local valueStr = encodeJson(v, keyStr) - -- Only include fields that have non-nil values - if valueStr ~= nil then - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. valueStr) - end - end - -- If no items were added, return nil to skip empty objects - if #items == 0 then - return nil - end - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - --- Get default mapping for event type -function getDefaultMapping(eventType) - local defaultMapping = { - ["user.session.start"] = { - {source = "uuid", target = "metadata.uid"}, - {source = "published", target = "metadata.original_time"}, - {source = "version", target = "api.version"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "authenticationContext.externalSessionId", target = "session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.authenticationProvider", target = "service.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.origin", target = "device.hostname"}, - {source = "debugContext.debugData.risk", target = "device.risk_level"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"} - }, - ["user.authentication.sso"] = { - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "authenticationContext.externalSessionId", target = "session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.authenticationProvider", target = "service.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.signOnMode", target = "auth_protocol"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"}, - -- target fields - {source = "dst_endpoint_uid", target = "dst_endpoint.uid"}, - {source = "actor_user_uid", target = "actor.user.uid"}, - {source = "dst_endpoint_svc_name", target = "dst_endpoint.svc_name"}, - {source = "actor_email_addr", target = "actor.email_addr"} - --{source = "actor_user_name", target = "user.name"} - }, - ["user.lifecycle.activate"] = { - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.browser", target = "client.userAgent.browser"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"} - -- target - --{source = "user_id", target = "user.id"} - --{source = "user_email_addr", target = "user.email_addr"} - --{source = "user_name", target = "user.name"} - }, - ["user.lifecycle.create"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestId", target = "api.request.uid"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"} - -- target - --{source = "user_id", target = "user.id"} - --{source = "user_email_addr", target = "user.email_addr"}, - --{source = "user_name", target = "user.name"} - }, - ["user.lifecycle.deactivate"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.requestId", target = "api.request.uid"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"} - -- target - --{source = "user_id", target = "user.id"}, - --{source = "user_email_addr", target = "user.email_addr"}, - --{source = "user_name", target = "user.name"} - }, - ["policy.evaluate_sign_on"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "authenticationContext.authenticationProvider", target = "name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "authenticationContext.externalSessionId", target = "session.uid"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "debugContext.debugData.deviceFingerprint", target = "device.uid"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.risk", target = "device.risk_level"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - {source = "version", target = "api.version"}, - -- target fields - {source = "actor.authorization.policy.uid", target = "actor.authorization.policy.uid"}, - {source = "actor.authorization.policy.name", target = "actor.authorization.policy.name"}, - {source = "actor.authorization.policy.rule.uid", target = "actor.authorization.policy.rule.uid"}, - {source = "actor.authorization.policy.rule.name", target = "actor.authorization.policy.rule.name"} - }, - ["system.org.rate_limit.warning"] = { - {source = "uuid", target = "metadata.uid"}, - {source = "published", target = "metadata.original_time"}, - {source = "version", target = "api.version"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"} - }, - ["application.user_membership.add"] = { - {source = "uuid", target = "metadata.uid"}, - {source = "published", target = "metadata.original_time"}, - {source = "version", target = "api.version"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "client.ipAddress", target = "src_endpoint.ip"}, - {source = "client.id", target = "src_endpoint.uid"}, - {source = "client.geographicalContext.city", target = "src_endpoint.location.city"}, - {source = "client.geographicalContext.country", target = "src_endpoint.location.country"}, - {source = "client.geographicalContext.geolocation.lat", target = "location.coordinates.lat"}, - {source = "client.geographicalContext.geolocation.lon", target = "location.coordinates.lon"}, - {source = "client.geographicalContext.postalCode", target = "src_endpoint.location.postal_code"}, - {source = "client.geographicalContext.state", target = "src_endpoint.location.region"}, - {source = "client.userAgent.rawUserAgent", target = "http_request.user_agent"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "authenticationContext.externalSessionId", target = "actor.session.uid"}, - {source = "authenticationContext.issuer.id", target = "actor.idp.uid"}, - {source = "authenticationContext.issuer.type", target = "actor.idp.name"}, - {source = "authenticationContext.credentialProvider", target = "actor.invoked_by"}, - {source = "authenticationContext.credentialType", target = "actor.user.account.type"}, - {source = "debugContext.debugData.requestUri", target = "http_request.url.path"}, - {source = "debugContext.debugData.url", target = "http_request.url.query_string"}, - {source = "debugContext.debugData.requestId", target = "http_request.uid"}, - {source = "debugContext.debugData.origin", target = "device.hostname"}, - {source = "debugContext.debugData.risk", target = "device.risk_level"}, - {source = "securityContext.isp", target = "src_endpoint.location.isp"}, - {source = "securityContext.domain", target = "src_endpoint.location.domain"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - -- target fields - {source = "dst_endpoint_uid", target = "dst_endpoint.uid"}, - {source = "dst_endpoint_svc_name", target = "dst_endpoint.svc_name"}, - {source = "actor_user_uid", target = "actor.user.uid"}, - {source = "actor_email_addr", target = "actor.email_addr"} - --{source = "actor_user_name", target = "user.name"} - }, - ["application.lifecycle.update"] = { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.type", target = "actor.user.type"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"}, - -- target fields - {source = "dst_endpoint_uid", target = "dst_endpoint.uid"}, - {source = "dst_endpoint_svc_name", target = "dst_endpoint.svc_name"}, - - --{source = "actor_user_uid", target = "actor.user.uid"}, - {source = "actor_email_addr", target = "actor.email_addr"} - --{source = "actor_user_name", target = "user.name"} - } - } - - return defaultMapping[eventType] or getGenericOktaMapping() -end - --- Generic Okta mapping -function getGenericOktaMapping() - return { - {source = "actor.id", target = "actor.user.uid"}, - {source = "actor.alternateId", target = "actor.user.email_addr"}, - {source = "actor.displayName", target = "actor.user.name"}, - {source = "outcome.result", target = "status"}, - {source = "outcome.reason", target = "status_detail"}, - {source = "published", target = "metadata.original_time"}, - {source = "transaction.id", target = "metadata.correlation_uid"}, - {source = "transaction.detail", target = "raw_data"}, - {source = "uuid", target = "metadata.uid"} - } -end - --- Common mapping -function getCommonMapping() - return { - {source = "category_name", target = "category_name"}, - {source = "category_uid", target = "category_uid"}, - {source = "class_uid", target = "class_uid"}, - {source = "severity_id", target = "severity_id"}, - {source = "activity_name", target = "activity_name"}, - {source = "activity_id", target = "activity_id"}, - {source = "type_uid", target = "type_uid"}, - {source = "vendor_name", target = "metadata.product.vendor_name"}, - {source = "name", target = "metadata.product.name"}, - {source = "OCSF_version", target = "metadata.version"}, - {source = "time", target = "time"}, - {source = "status_id", target = "status_id"}, - {source = "type_id", target = "actor.user.type_id"}, - {source = "observables", target = "observables"}, - {source = "dataSource.category", target = "dataSource.category"}, - {source = "site.id", target = "site.id"}, - {source = "dataSource.name", target = "dataSource.name"}, - {source = "dataSource.vendor", target = "dataSource.vendor"}, - {source = "message", target = "message"}, - {source = "class_name", target = "class_name"}, - {source = "type_name", target = "type_name"} - } -end - --- Parse security domain -function parseSecurityDomain(parsedLog) - if parsedLog["src_endpoint.location.domain"] == "." then - parsedLog["src_endpoint.location.domain"] = nil - end - return parsedLog -end - --- Set site ID -function setSiteId(oktaLog, siteId) - oktaLog["site"] = {id = siteId} - return oktaLog -end - --- Parse risk level -function parseRiskLevel(parsedLog) - local riskLevelMapper = { - ["Info"] = 0, ["Low"] = 1, ["Medium"] = 2, - ["High"] = 3, ["Critical"] = 4 - } - local riskPrefix = "level=" - local finalRiskLevel = "Other" - local finalRiskLevelId = 99 - - -- Look for risk level in the already-mapped device.risk_level field - local riskLevel = "" - if parsedLog["device"] and parsedLog["device"]["risk_level"] then - riskLevel = parsedLog["device"]["risk_level"] - end - - for riskLevelName, riskLevelId in pairs(riskLevelMapper) do - if string.find(string.lower(riskLevel), string.lower(riskPrefix .. riskLevelName)) then - finalRiskLevel = riskLevelName - finalRiskLevelId = riskLevelId - break - end - end - - -- Set the parsed risk level back to the device object - if not parsedLog["device"] then parsedLog["device"] = {} end - parsedLog["device"]["risk_level"] = finalRiskLevel - parsedLog["device"]["risk_level_id"] = finalRiskLevelId - return parsedLog -end - --- Find event type -function findEventType(log) - local eventType = log["eventType"] - if eventType == nil then - return "unknown" - end - return eventType -end - --- Get category UID -function getCategoryUID(eventType) - local categoryMapping = { - ["user.session.start"] = 3, - ["user.authentication.sso"] = 3, - ["user.lifecycle.activate"] = 2, - ["user.lifecycle.create"] = 3, - ["user.lifecycle.deactivate"] = 3, - ["policy.evaluate_sign_on"] = 3, - ["system.org.rate_limit.warning"] = 3, - ["application.user_membership.add"] = 3, - ["application.lifecycle.update"] = 6 - } - return categoryMapping[eventType] or 0 -end - --- Get class mapping -function getClassMapping(eventType) - local classMapping = { - ["user.session.start"] = {name = "Authentication", id = 3002}, - ["user.authentication.sso"] = {name = "Authentication", id = 3002}, - ["user.lifecycle.activate"] = {name = "Account Change", id = 3001}, - ["user.lifecycle.create"] = {name = "Account Change", id = 3001}, - ["user.lifecycle.deactivate"] = {name = "Account Change", id = 3001}, - ["policy.evaluate_sign_on"] = {name = "Authentication", id = 3002}, - ["system.org.rate_limit.warning"] = {name = "API Activity", id = 6003}, - ["application.user_membership.add"] = {name = "Account Change", id = 3001}, - ["application.lifecycle.update"] = {name = "Application Lifecycle", id = 6002} - } - local mapping = classMapping[eventType] or {name = "Base Event", id = 0} - return mapping.name, mapping.id -end - --- Get status default OCSF mapping -function getStatusDefaultOCSFMapping(status) - if status == nil then - return 0 - end - - local statusMapping = { - ["SUCCESS"] = 1, - ["FAILURE"] = 2 - } - return statusMapping[status] or 99 -end - --- Get activity name -function getActivityName(eventType, eventTypeDisplayName) - local activityMapping = { - ["user.session.start"] = {name = "Logon", id = 1}, - ["user.authentication.sso"] = {name = "Logon", id = 1}, - ["user.lifecycle.activate"] = {name = "Enable", id = 2}, - ["user.lifecycle.create"] = {name = "Create", id = 1}, - ["user.lifecycle.deactivate"] = {name = "Disable", id = 5}, - ["policy.evaluate_sign_on"] = {name = "Logon", id = 1}, - ["application.user_membership.add"] = {name = "Attach Policy", id = 7} - } - local mapping = activityMapping[eventType] or {name = eventTypeDisplayName, id = 99} - return mapping.name, mapping.id -end - --- Get user type -function getUserType(oktaUserType) - local userTypeMapping = { - ["Unknown"] = 0, - ["User"] = 1, - ["Admin"] = 2, - ["System"] = 3, - ["Other"] = 99 - } - for user, id in pairs(userTypeMapping) do - if string.find(oktaUserType or "", user) then - return id - end - end - return 99 -end - --- Get category mapper -function getCategoryMapper(eventType) - local categoryMapper = { - ["user.session.start"] = "Identity & Access Management", - ["user.authentication.sso"] = "Identity & Access Management", - ["user.lifecycle.activate"] = "Identity & Access Management", - ["user.lifecycle.create"] = "Identity & Access Management", - ["user.lifecycle.deactivate"] = "Identity & Access Management", - ["policy.evaluate_sign_on"] = "Identity & Access Management", - ["system.org.rate_limit.warning"] = "Application Activity", - ["application.user_membership.add"] = "Identity & Access Management", - ["application.lifecycle.update"] = "Application Activity" - } - return categoryMapper[eventType] or "Uncategorized" -end - --- Get type name -function getTypeName(eventType) - local typeMapper = { - ["user.session.start"] = "Authentication: Logon", - ["user.authentication.sso"] = "Authentication: Logon", - ["user.lifecycle.activate"] = "Account Change: Enable", - ["user.lifecycle.create"] = "Account Change: Create", - ["user.lifecycle.deactivate"] = "Account Change: Disable", - ["policy.evaluate_sign_on"] = "Authentication: Logon", - ["system.org.rate_limit.warning"] = "API Activity: Other", - ["application.user_membership.add"] = "Account Change: Attach Policy", - ["application.lifecycle.update"] = "Application Lifecycle: Other" - } - return typeMapper[eventType] or "Base Event: Other" -end - --- Get observables -function getObservables(log) - local observables = {} - - -- Hostname observable - local hostname = safelyAccessNestedDictKeys({"debugContext", "debugData", "origin"}, log) - if hostname and hostname ~= "" and hostname ~= "null" then - table.insert(observables, { - type_id = 1, - type = "Hostname", - name = "device.hostname", - value = hostname - }) - end - - -- IP Address observable - local clientIpAddress = safelyAccessNestedDictKeys({"client", "ipAddress"}, log) - if clientIpAddress and clientIpAddress ~= "" and clientIpAddress ~= "null" then - table.insert(observables, { - type_id = 2, - type = "IP Address", - name = "src_endpoint.ip", - value = clientIpAddress - }) - end - - -- User Name observable - local userName = safelyAccessNestedDictKeys({"actor", "displayName"}, log) - if userName and userName ~= "" and userName ~= "null" then - table.insert(observables, { - type_id = 4, - type = "User Name", - name = "actor.user.name", - value = userName - }) - end - - -- Email Address observable - local emailAddress = safelyAccessNestedDictKeys({"actor", "alternateId"}, log) - if emailAddress and emailAddress ~= "" and emailAddress ~= "null" then - table.insert(observables, { - type_id = 5, - type = "Email Address", - name = "actor.user.email_addr", - value = emailAddress - }) - end - - -- URL String observable - local requestUri = safelyAccessNestedDictKeys({"debugContext", "debugData", "requestUri"}, log) - if requestUri and requestUri ~= "" and requestUri ~= "null" then - table.insert(observables, { - type_id = 6, - type = "URL String", - name = "http_request.url.path", - value = requestUri - }) - end - - -- Geo Location observable - local lat = safelyAccessNestedDictKeys({"client", "geographicalContext", "geolocation", "lat"}, log) - local lon = safelyAccessNestedDictKeys({"client", "geographicalContext", "geolocation", "lon"}, log) - local notAllowedItems = {{}, {}, "", nil} - local latAllowed = true - local lonAllowed = true - for _, item in ipairs(notAllowedItems) do - if lat == item then latAllowed = false end - if lon == item then lonAllowed = false end - end - if latAllowed and lonAllowed and lat and lon then - table.insert(observables, { - type_id = 26, - type = "Geo Location", - name = "client.geographicalContext.geolocation", - value = lat .. ", " .. lon - }) - end - - return observables -end - --- Process target fields into flat keys for mapping -function processTargetFields(log, eventType) - local targetFields = {} - - if eventType == "user.authentication.sso" or - eventType == "application.user_membership.add" or - eventType == "application.lifecycle.update" then - local target = log["target"] - if target and type(target) == "table" then - for _, t in ipairs(target) do - if type(t) == "table" then - if t["type"] == "AppInstance" then - log["dst_endpoint_uid"] = t["type"] - log["dst_endpoint_svc_name"] = t["displayName"] - elseif t["type"] == "AppUser" then - -- Override specific fields with target values - log["actor_user_uid"] = t["type"] -- "AppUser" overrides actor.id - log["actor_email_addr"] = t["alternateId"] -- "unknown" for actor.email_addr - log["actor_user_name"] = t["displayName"] -- "John Cena" (same as original) - - -- Override actor.id for actor.user.uid mapping - if log["actor"] then - log["actor"]["id"] = t["type"] -- "AppUser" for actor.user.uid - -- Keep actor.alternateId as original for actor.user.email_addr mapping - -- Don't override actor.alternateId - let it stay as "john.cena@wwe.com" - end - end - end - end - end - elseif eventType == "user.lifecycle.activate" or - eventType == "user.lifecycle.create" or - eventType == "user.lifecycle.deactivate" then - local target = log["target"] - if target and type(target) == "table" and #target > 0 then - local t = target[1] - if type(t) == "table" then - targetFields["user_id"] = t["id"] - targetFields["user_email_addr"] = t["alternateId"] - targetFields["user_name"] = t["displayName"] - end - end - elseif eventType == "policy.evaluate_sign_on" then - local target = log["target"] - if target and type(target) == "table" then - for _, t in ipairs(target) do - if type(t) == "table" then - if t["type"] == "PolicyEntity" then - targetFields["actor"] = { - authorization = { - policy = { - name = t["displayName"], - uid = t["id"] - } - } - } - elseif t["type"] == "PolicyRule" then - targetFields["actor"] = { - authorization = { - policy = { - rule = { - uid = t["id"], - name = t["displayName"] - } - } - } - } - end - end - end - end - end - - return targetFields -end - --- Generate severity mapping -function generateSeverityMapping(availableSeverityList) - local defaultSeverityMapping = { - ["DEBUG"] = 0, ["INFO"] = 1, ["WARN"] = 3, ["ERROR"] = 5, ["OTHER"] = 99 - } - local defaultSeverityMappingKeys = {"DEBUG", "INFO", "WARN", "ERROR", "OTHER"} - local severityIDMapping = {} - - for severityTypeIndex = 1, #availableSeverityList do - if availableSeverityList[severityTypeIndex] then - local key = defaultSeverityMappingKeys[severityTypeIndex] - severityIDMapping[key] = defaultSeverityMapping[key] - end - end - return severityIDMapping -end - --- Get severity ID -function getSeverityID(eventType, logSeverity) - local severityMapping = { - ["user.session.start"] = generateSeverityMapping({true, true, true, true, true}), - ["user.authentication.sso"] = generateSeverityMapping({true, true, true, false, false}), - ["user.lifecycle.activate"] = generateSeverityMapping({true, true, true, false, false}), - ["user.lifecycle.create"] = generateSeverityMapping({false, true, false, false, false}), - ["user.lifecycle.deactivate"] = generateSeverityMapping({false, true, false, false, false}), - ["policy.evaluate_sign_on"] = generateSeverityMapping({false, true, false, false, false}), - ["system.org.rate_limit.warning"] = generateSeverityMapping({true, true, true, false, false}), - ["application.user_membership.add"] = generateSeverityMapping({true, true, true, false, false}), - ["application.lifecycle.update"] = generateSeverityMapping({false, true, false, false, false}) - } - local eventSeverityMapping = severityMapping[eventType] or {} - return eventSeverityMapping[logSeverity] or 99 -end - --- Generate synthetic fields -function generateSyntheticFields(log, eventType, originalLog) - -- Set nested metadata fields - if not log["metadata"] then log["metadata"] = {} end - if not log["metadata"]["product"] then log["metadata"]["product"] = {} end - log["metadata"]["product"]["vendor_name"] = "Okta" - log["metadata"]["product"]["name"] = "Okta" - log["metadata"]["version"] = "1.0.0" - - local publishedTime = safelyAccessNestedDictKeys({"published"}, originalLog) - if publishedTime then - log["time"] = convertToMilliseconds(publishedTime) - end - - -- Use the event type passed as parameter instead of finding it again - log["category_name"] = getCategoryMapper(eventType) - log["category_uid"] = getCategoryUID(eventType) - - local className, classUid = getClassMapping(eventType) - log["class_name"] = className - log["class_uid"] = classUid - - -- Dynamic severity calculation - log["severity_id"] = getSeverityID(eventType, originalLog["severity"]) - - local activityName, activityId = getActivityName(eventType, originalLog["displayMessage"] or "Other") - log["activity_name"] = activityName - log["activity_id"] = activityId - - log["type_uid"] = (classUid * 100) + activityId - log["type_name"] = getTypeName(eventType) - - local outcomeResult = safelyAccessNestedDictKeys({"outcome", "result"}, originalLog) - log["status_id"] = getStatusDefaultOCSFMapping(outcomeResult) - - local actorType = safelyAccessNestedDictKeys({"actor", "type"}, originalLog) - - if not log["actor"] then log["actor"] = {} end - if not log["actor"]["user"] then log["actor"]["user"] = {} end - log["actor"]["user"]["type_id"] = getUserType(actorType) - log["actor"]["user"]["type"] = actorType or "User" - -- Actor email_addr is now set by target field mappings, don't override - - -- Handle postal code conversion to string - local postalCode = safelyAccessNestedDictKeys({"client", "geographicalContext", "postalCode"}, log) - if postalCode then - if not log["client"] then log["client"] = {} end - if not log["client"]["geographicalContext"] then log["client"]["geographicalContext"] = {} end - log["client"]["geographicalContext"]["postalCode"] = tostring(postalCode) - end - - -- Target fields are now processed earlier in oktaLogsMapping - - - log["event.type"] = log["activity_name"] or eventType - log["dataSource"] = {name = "Okta", category = "security", vendor = "Okta"} - - -- Add session field (skip root session for lifecycle events except deactivate) - local sessionId = safelyAccessNestedDictKeys({"authenticationContext", "externalSessionId"}, originalLog) - if not (eventType == "user.lifecycle.activate" or eventType == "user.lifecycle.create") then - if eventType == "user.lifecycle.deactivate" then - -- For deactivate events, add session to existing actor object if it exists - if log["actor"] then - log["actor"]["session"] = {uid = sessionId or "unknown"} - else - log["session"] = {uid = sessionId or "unknown"} - end - end - end - - -- Add user field from actor only for non-lifecycle events - --local userName = safelyAccessNestedDictKeys({"actor", "displayName"}, originalLog) - --if userName and not (eventType == "user.lifecycle.activate" or eventType == "user.lifecycle.create" or eventType == "user.lifecycle.deactivate" or eventType == "user.session.start") then - -- log["user"] = {name = userName} - --end - - return log -end - - --- Helper function to check if a field should be ignored -function shouldIgnoreField(fieldName) - - local ignoreFields = { - "_okta_event_type", "_ob", "ts", "timestamp" - } - - for _, field in ipairs(ignoreFields) do - if fieldName == field or string.find(fieldName, "^" .. field .. "%.") then - return true - end - end - return false -end - --- Helper function to check if a value is empty (empty object, array, or null) -function isEmptyValue(value) - if value == nil or value == "NULL_PLACEHOLDER" then - return true - end - if type(value) == "string" and (value == "" or value == "null") then - return true - end - if type(value) == "table" then - -- Check if it's an empty table - if next(value) == nil then - return true - end - -- For unmapped fields, recursively check nested values to filter out empty nested objects - local hasNonEmptyValues = false - for k, v in pairs(value) do - if not isEmptyValue(v) then - hasNonEmptyValues = true - break - end - end - return not hasNonEmptyValues - end - return false -end - --- Helper function to add unmapped fields as a truly nested object (no dotted keys) -function addUnmappedFields(sourceObj, targetObj, mappedFields, prefixParts) - prefixParts = prefixParts or {} - for key, value in pairs(sourceObj) do - -- Skip ignored fields - if shouldIgnoreField(key) then - goto continue - end - - -- Build current dotted path for mapped check - local currentPathParts = {} - for i = 1, #prefixParts do currentPathParts[i] = prefixParts[i] end - table.insert(currentPathParts, key) - local currentPath = table.concat(currentPathParts, ".") - - -- Check if this exact path has been mapped - local isMapped = mappedFields[currentPath] or false - - if not isMapped and value ~= nil then - if type(value) == "table" then - local nestedObj = {} - addUnmappedFields(value, nestedObj, mappedFields, currentPathParts) - - -- Check if ALL direct children were mapped - local allChildrenMapped = true - for childKey, childValue in pairs(value) do - local childPath = currentPath .. "." .. childKey - if not mappedFields[childPath] then - allChildrenMapped = false - break - end - end - - -- Only add parent if it has unmapped children OR no children were mapped - if next(nestedObj) and not allChildrenMapped then - targetObj[key] = nestedObj - end - -- Don't add empty objects to unmapped - else - -- Only add non-empty values to unmapped - if not isEmptyValue(value) then - targetObj[key] = value - end - end - end - ::continue:: - end -end - --- Helper function to build nested object structure from flat dotted keys -function buildNestedStructure(flatObj) - local nested = {} - for key, value in pairs(flatObj) do - local keys = split(key, ".") - local current = nested - for i = 1, #keys - 1 do - local k = keys[i] - if not current[k] then - current[k] = {} - elseif type(current[k]) ~= "table" then - -- If the existing value is not a table, we can't create nested structure - -- Skip this key to avoid conflicts - goto continue - end - current = current[k] - end - - -- Special handling for raw_data field - encode as JSON string - if keys[#keys] == "raw_data" then - -- For raw_data, we want the value as a JSON string, not double-encoded - if type(value) == "table" then - current[keys[#keys]] = encodeJson(value, "raw_data") - else - -- If it's already a string, use it as-is - current[keys[#keys]] = tostring(value) - end - else - current[keys[#keys]] = value - end - ::continue:: - end - return nested -end - --- Helper function to set nested value (marks field as processed) -function setNestedValue(obj, keys, value) - local current = obj - for i = 1, #keys - 1 do - if not current[keys[i]] then - current[keys[i]] = {} - end - current = current[keys[i]] - end - current[keys[#keys]] = value -end - - --- Helper function to filter out ignored fields using shouldIgnoreField -function filterIgnoredFields(log) - local function filterObject(obj, prefix) - prefix = prefix or "" - local filteredObj = {} - - for key, value in pairs(obj) do - local fullKey = prefix == "" and key or prefix .. "." .. key - - if not shouldIgnoreField(fullKey) then - if type(value) == "table" then - local filteredValue = filterObject(value, fullKey) - if next(filteredValue) then -- Only add non-empty tables - filteredObj[key] = filteredValue - end - else - filteredObj[key] = value - end - end - end - return filteredObj - end - - return filterObject(log) -end - - --- Helper function to create ordered JSON message using FIELD_ORDER approach -function createOrderedMessage(log, eventType) - -- Filter out null values and empty strings from the log before creating message - local filteredLog = {} - for k, v in pairs(log) do - if v ~= nil and v ~= "" and v ~= "null" then - if type(v) == "table" then - local filteredValue = filterNullValues(v) - if next(filteredValue) then -- Only add non-empty tables - filteredLog[k] = filteredValue - end - else - filteredLog[k] = v - end - end - end - - -- Apply domain filtering to the message for lifecycle events only - if eventType == "user.lifecycle.create" or eventType == "user.lifecycle.deactivate" or eventType == "user.session.start" then - filteredLog = applyDomainFiltering(filteredLog) - end - - -- Use FIELD_ORDERS.root to create ordered JSON string - local orderedJson = encodeWithFieldOrder(filteredLog, FIELD_ORDERS.root) - return orderedJson or "{}" -end - --- Helper function to recursively filter out null values from nested tables -function filterNullValues(obj) - local filtered = {} - for k, v in pairs(obj) do - if v ~= nil and v ~= "" and v ~= "null" then - if type(v) == "table" then - local filteredValue = filterNullValues(v) - if next(filteredValue) then -- Only add non-empty tables - filtered[k] = filteredValue - end - else - filtered[k] = v - end - end - end - return filtered -end - --- Helper function to apply domain filtering recursively -function applyDomainFiltering(obj) - if type(obj) ~= "table" then - return obj - end - - local filtered = {} - for k, v in pairs(obj) do - if type(v) == "table" then - local filteredValue = applyDomainFiltering(v) - if next(filteredValue) then -- Only add non-empty tables - filtered[k] = filteredValue - end - else - -- Skip domain fields with value "." and isp fields that duplicate asOrg - if k == "domain" and v == "." then - -- Skip this field - elseif k == "isp" and obj["asOrg"] and v == obj["asOrg"] then - -- Skip isp field when it's the same as asOrg - else - filtered[k] = v - end - end - end - return filtered -end - --- Simplified Okta logs mapping function -function oktaLogsMapping(log) - -- STEP 1: First filter out ignored fields using shouldIgnoreField - log = filterIgnoredFields(log) - - -- STEP 2: Find event type first - local eventType = findEventType(log) - - -- STEP 3: Create ordered JSON message from original log BEFORE any modifications - local originalLog = {} - for k, v in pairs(log) do - originalLog[k] = v - end - log.message = createOrderedMessage(originalLog, eventType) - - -- STEP 4: Process target fields to create flat keys for mapping - local targetFields = processTargetFields(log, eventType) - - -- Apply target fields to log for specific events so mappings can consume them - if eventType == "policy.evaluate_sign_on" then - for key, value in pairs(targetFields) do - if key == "actor" then - if not log["actor"] then log["actor"] = {} end - for subKey, subValue in pairs(value) do - log["actor"][subKey] = subValue - end - else - log[key] = value - end - end - elseif eventType == "user.lifecycle.activate" or eventType == "user.lifecycle.create" or eventType == "user.lifecycle.deactivate" then - -- For lifecycle events, expose target-derived user fields on the log - for key, value in pairs(targetFields) do - log[key] = value - end - end - - -- STEP 4: Get mapping for event type - local mappings = getDefaultMapping(eventType) - - -- Merge with common mapping - local commonMappings = getCommonMapping() - for _, mapping in ipairs(commonMappings) do - table.insert(mappings, mapping) - end - - -- STEP 4: Apply mappings and track processed fields - local parsedData = {} - local mappedFields = {} - for _, mapping in ipairs(mappings) do - local sourcePath = mapping.source - local targetPath = mapping.target - local keys = split(sourcePath, ".") - local value = safelyAccessNestedDictKeys(keys, log) - - if value ~= nil and value ~= "" and value ~= "null" then - if targetPath == "src_endpoint.location.domain" and value == "." then - -- Skip this field for any event when domain is a single dot - else - -- Update target in parsedData - parsedData[targetPath] = value - -- Track mapped field by source path - mappedFields[sourcePath] = true - end - end - end - - -- STEP 5: Add remaining non-null fields to unmapped - parsedData["unmapped"] = {} - local ignoreUnmappedFields = { - "actor_user_uid", - "actor_email_addr", - "actor_user_name", - "user_id", - "user_email_addr", - "user_name" - } - for _, field in ipairs(ignoreUnmappedFields) do - mappedFields[field] = true - end - - addUnmappedFields(log, parsedData["unmapped"], mappedFields, {}) - - -- Apply domain filtering to unmapped fields as well - parsedData["unmapped"] = applyDomainFiltering(parsedData["unmapped"]) - - -- STEP 6: Convert flat dotted keys to nested objects - parsedData = buildNestedStructure(parsedData) - - -- STEP 7: Apply post-processing - parsedData = parseSecurityDomain(parsedData) - parsedData = parseRiskLevel(parsedData) - - -- STEP 8: Generate synthetic fields - parsedData = generateSyntheticFields(parsedData, eventType, log) - - -- STEP 9: Generate observables for specific event types (after synthetic fields) - local observablesEventTypes = { - "user.authentication.sso", - "application.user_membership.add", - "user.lifecycle.activate", - "user.lifecycle.create", - "user.lifecycle.deactivate", - "policy.evaluate_sign_on", - "system.org.rate_limit.warning", - "user.session.start" - } - - for _, eventTypeName in ipairs(observablesEventTypes) do - if eventType == eventTypeName then - local observables = getObservables(log) - -- Convert observables to array format - local observablesArray = {} - for _, obs in ipairs(observables) do - table.insert(observablesArray, obs) - end - parsedData["observables"] = observablesArray - break - end - end - - parsedData = convertToNested(parsedData, eventType) - return parsedData -end - --- Convert flat parsed data to nested JSON structure with field ordering -function convertToNested(parsedData, eventType) - local nested = {} - - -- First, build the nested structure - for key, value in pairs(parsedData) do - -- Special case: keep event.type as flattened field (don't nest it) - if key == "event.type" then - nested["event.type"] = value -- Keep as flattened event.type - goto continue - end - - local keys = split(key, ".") - local current = nested - - -- Navigate to the parent object - for i = 1, #keys - 1 do - local k = keys[i] - if not current[k] then - current[k] = {} - elseif type(current[k]) ~= "table" then - -- If the existing value is not a table, we can't create nested structure - -- Skip this key to avoid conflicts - goto continue - end - current = current[k] - end - - -- Set the final value - special handling for raw_data field - if keys[#keys] == "raw_data" then - -- For raw_data, we want the value as a JSON string, not double-encoded - if type(value) == "table" then - current[keys[#keys]] = encodeJson(value, "raw_data") - else - -- If it's already a string, use it as-is - current[keys[#keys]] = tostring(value) - end - else - current[keys[#keys]] = value - end - ::continue:: - end - - -- Then, apply field ordering to the nested structure - return applyFieldOrdering(nested, nil, eventType) -end - --- Apply field ordering to nested structure -function applyFieldOrdering(obj, fieldOrder, eventType) - fieldOrder = fieldOrder or FIELD_ORDERS.root - local ordered = {} - - -- Phase 1: Add fields in predefined order - for _, fieldName in ipairs(fieldOrder) do - if obj[fieldName] ~= nil then - if type(obj[fieldName]) == "table" then - -- Recursively apply ordering to nested objects - local nestedFieldOrder = FIELD_ORDERS[fieldName] - ordered[fieldName] = applyFieldOrdering(obj[fieldName], nestedFieldOrder, eventType) - else - ordered[fieldName] = obj[fieldName] - end - end - end - - -- Phase 2: Add remaining fields not in the predefined order - for key, value in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if key == fieldName then - found = true - break - end - end - - if not found then - if type(value) == "table" then - -- Recursively apply ordering to nested objects - local nestedFieldOrder = FIELD_ORDERS[key] - ordered[key] = applyFieldOrdering(value, nestedFieldOrder, eventType) - else - ordered[key] = value - end - end - end - - return ordered -end - --- Convert ISO 8601 timestamp to Unix epoch milliseconds -function convertToMilliseconds(timestamp) - if not timestamp or timestamp == "" then - return nil - end - - -- Parse ISO 8601 format: "2025-09-29T09:15:40Z" or "2025-09-29T09:15:40.123Z" - local year, month, day, hour, min, sec, ms = string.match(timestamp, "(%d+)%-(%d+)%-(%d+)T(%d+):(%d+):(%d+)%.?(%d*)Z") - - if year and month and day and hour and min and sec then - local t = { - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec), - isdst = false - } - - -- Get local time interpretation - local local_seconds = os.time(t) - - -- Get what this represents in UTC - local utc_date = os.date("!*t", local_seconds) - - local unix_seconds - -- Check if the UTC interpretation matches our input - if utc_date.year == tonumber(year) and utc_date.month == tonumber(month) and - utc_date.day == tonumber(day) and utc_date.hour == tonumber(hour) and - utc_date.min == tonumber(min) and utc_date.sec == tonumber(sec) then - -- We are already in UTC, use as-is - unix_seconds = local_seconds - else - -- Calculate the correct UTC timestamp - local utc_seconds = os.time(utc_date) - local offset = local_seconds - utc_seconds - unix_seconds = local_seconds + offset -- Add offset to get UTC - end - - -- Add milliseconds if present - local milli = 0 - if ms and ms ~= "" then - milli = tonumber((ms .. "000"):sub(1, 3)) -- pad/truncate to 3 digits - end - - return unix_seconds * 1000 + milli - end - - return nil -end - --- Main event processing function -function processEvent(event) - return oktaLogsMapping(event) -end \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/wiz_issue/metadata.yaml b/pipelines/community/transform_ocsf/wiz_issue/metadata.yaml deleted file mode 100644 index 9699c5a..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/metadata.yaml +++ /dev/null @@ -1,52 +0,0 @@ -grade: - letter: D - score: 60 - verdict: analyzer_limit - required_field_coverage_pct: 0 - source_field_coverage_pct: 100 - tested_with_realistic_event: true - validated_at: '2026-04-19' -metadata_details: - purpose: OCSF-compliant Lua serializer for Wiz Issue. Maps source events to OCSF unclassified (class_uid=n/a) - following the processEvent contract. - datasource_vendor: wiz - dataSource: Wiz Issue - format: source-specific JSON/KV/syslog - ocsf_version: 1.3.0 - ingestion_method: Observo OCSFSerializer (Lua-based transform) - ingest_mode: "API Call" - auth_type: "API Key & Secret" - sample_record: "{\n \"id\": \"wiz-issue-92324e96-d8d1-4e9a-bcb8-14eed356e1a5\",\n \"targetExternalId\"\ - : \"arn:aws:s3:::company-public-assets\",\n \"deleted\": false,\n \"targetObjectProviderUniqueId\"\ - : \"arn:aws:s3:::company-public-assets\",\n \"firstSeenAt\": \"2026-04-19T14:51:33.000Z\",\n \"\ - result\": \"FAIL\",\n \"status\": \"REJECTED\",\n \"severity\": \"MEDIUM\",\n \"remediation\":\ - \ \"Update bucket ACL to remove public access\",\n \"resource\": {\n \"id\": \"resource-8b1a9118-bd96-48fa-9661-c8fef99bc28c\"\ - ,\n \"providerId\": \"aws-account-123456789012\",\n \"name\": \"company-public-assets\",\n \ - \ \"nativeType\": \"S3 Bucket\",\n \"type\": \"BUCKET\",\n \"region\": \"ap-northeast-1\"\ - ,\n \"subscription\": {\n \"id\": \"sub-aws-prod-001\",\n \"name\": \"AWS Production\ - \ Account\",\n \"externalId\": \"123456789012\",\n \"cloudProvider\": \"AWS\"\n },\n\ - \ \"projects\": [\n {\n \"id\": \"project-marketing-001\",\n \"name\": \"Marketing\ - \ Assets\",\n \"riskProfile\": {\n \"businessImpact\": \"MBI\"\n }\n }\n\ - \ ],\n \"tags\": [\n {\n \"key\": \"Environment\",\n \"value\": \"Production\"\ - \n },\n {\n \"key\": \"Department\",\n \"value\": \"Marketing\"\n }\n\ - \ ]\n },\n \"rule\": {\n \"id\": \"rule-s3-public-001\",\n \"graphId\": \"graph-rule-001\"\ - ,\n \"name\": \"S3 bucket is publicly accessible\",\n \"description\": \"S3 bucket allows public\ - \ read access\",\n \"remediationInstructions\": \"Remove public ACL grants from bucket policy\"\ - ,\n \"functionAsControl\": false\n },\n \"securitySubCategories\": [\n {\n \"id\":" - dependency_summary: Requires Observo OCSFSerializer template with Lua runtime (lupa). Source events - must match the field layout exercised by the Lua mappings. - performance_impact: Single-event transform, ~? lines. Field-for-field normalization with helper-based - nested access. - ocsf_mapping: - class_uid: null - class_name: null - category_uid: null - category_name: null - tags: wiz, ocsf, lua, serializer, transform - version: v1.0 - author: Community (imported from Observo platform UI) - validation: - harness_grade: D - harness_score: 60 - orion_reviewed: true - orion_verdict: signed_off diff --git a/pipelines/community/transform_ocsf/wiz_issue/sample.json b/pipelines/community/transform_ocsf/wiz_issue/sample.json deleted file mode 100644 index 5666680..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/sample.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "id": "wiz-issue-92324e96-d8d1-4e9a-bcb8-14eed356e1a5", - "targetExternalId": "arn:aws:s3:::company-public-assets", - "deleted": false, - "targetObjectProviderUniqueId": "arn:aws:s3:::company-public-assets", - "firstSeenAt": "2026-04-19T14:51:33.000Z", - "result": "FAIL", - "status": "REJECTED", - "severity": "MEDIUM", - "remediation": "Update bucket ACL to remove public access", - "resource": { - "id": "resource-8b1a9118-bd96-48fa-9661-c8fef99bc28c", - "providerId": "aws-account-123456789012", - "name": "company-public-assets", - "nativeType": "S3 Bucket", - "type": "BUCKET", - "region": "ap-northeast-1", - "subscription": { - "id": "sub-aws-prod-001", - "name": "AWS Production Account", - "externalId": "123456789012", - "cloudProvider": "AWS" - }, - "projects": [ - { - "id": "project-marketing-001", - "name": "Marketing Assets", - "riskProfile": { - "businessImpact": "MBI" - } - } - ], - "tags": [ - { - "key": "Environment", - "value": "Production" - }, - { - "key": "Department", - "value": "Marketing" - } - ] - }, - "rule": { - "id": "rule-s3-public-001", - "graphId": "graph-rule-001", - "name": "S3 bucket is publicly accessible", - "description": "S3 bucket allows public read access", - "remediationInstructions": "Remove public ACL grants from bucket policy", - "functionAsControl": false - }, - "securitySubCategories": [ - { - "id": "cat-data-exposure-001", - "title": "Data Exposure", - "category": { - "id": "cat-parent-001", - "name": "Data Protection", - "framework": { - "id": "framework-cis-001", - "name": "CIS AWS Foundations Benchmark" - } - } - } - ], - "ignoreRules": null -} \ No newline at end of file diff --git a/pipelines/community/transform_ocsf/wiz_issue/serializer.lua b/pipelines/community/transform_ocsf/wiz_issue/serializer.lua deleted file mode 100644 index 9a94bba..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/serializer.lua +++ /dev/null @@ -1,632 +0,0 @@ - -local FEATURES = { - FLATTEN_EVENT_TYPE = true, -} - --- Wiz to OCSF Mapping Script -local OCSF_VERSION = "v1.0.0-rc.3" - -local WIZ_SEVERITY_MAP = {["INFORMATIONAL"]=1, ["LOW"]=2, ["MEDIUM"]=3, ["HIGH"]=4, ["CRITICAL"]=5, ["OTHER"]=99} -local WIZ_STATE_MAP = {["OPEN"]=1, ["IN_PROGRESS"]=2, ["REJECTED"]=3, ["RESOLVED"]=4, ["OTHER"]=99} -local OCSF_ACCOUNT_TYPE_ID = {["AWS"]=10, ["GCP"]=5, ["EKS"]=10, ["GKE"]=5} -local OCSF_ACCOUNT_TYPE_NAME = {[10]="AWS Account", [5]="GCP Account", [3]="AWS IAM User", [4]="AWS IAM Role"} - --- Field ordering templates for consistent JSON serialization -local FIELD_ORDERS = { - root = {"activity_id", "analytic", "category_name", "category_uid", "class_name", "class_uid", - "cloud", "dataSource", "event", "finding", "index", "metadata", "message", "resource", - "severity", "severity_id", "state", "state_id", - "status", "type_uid", "unmapped" - }, - message = { - "activity_id", "analytic_type", "analytic_type_id", "category_name", "category_uid", - "class_name", "class_uid", "cloud_account_type", "cloud_account_type_id", "datasource", - "entitySnapshot", "event", "id", "ocsf_version", "severity", "severity_id", - "sourceRule", "state_id", "status", "type_uid", "updatedAt", "eventType", "readOnly", "createdAt" - }, - entitySnapshot = { - "cloudPlatform", "cloudProviderURL", "externalId", "id", "name", "nativeType", - "projects", "providerId", "region", "resourceGroupExternalId", "status", - "subscription", "subscriptionExternalId", "type", "wizResourceID" - }, - sourceRule = { - "description", "id", "name", "remediationInstructions", "resolutionRecommendation" - }, - datasource = { - "category", "name", "vendor" - }, - subscription = { - "externalId", "name", "id", "cloudProvider" - }, - projects = { - "name", "riskProfile", "id" - }, - riskProfile = { - "businessImpact" - }, - unmapped = { - "severity" - } -} - --- JSON encoding function (from AWS CloudTrail) -function encodeJson(obj, key) - if obj == nil or obj == "NULL_PLACEHOLDER" then - return "null" - elseif type(obj) == "boolean" then - return tostring(obj) - elseif type(obj) == "number" then - return tostring(obj) - elseif type(obj) == "string" then - return '"' .. obj:gsub('"', '\\"') .. '"' - elseif type(obj) == "table" then - local isArray = true - local maxIndex = 0 - for k, v in pairs(obj) do - if type(k) ~= "number" then - isArray = false - break - end - maxIndex = math.max(maxIndex, k) - end - - if isArray and maxIndex > 0 then - local items = {} - for i = 1, maxIndex do - local elementKey = key or tostring(i) - table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or "null") - end - return "[" .. table.concat(items, ", ") .. "]" - else - local items = {} - local fieldOrder = FIELD_ORDERS[key] or {} - - -- Phase 1: Process fields in predefined order - for _, fieldName in ipairs(fieldOrder) do - local v = obj[fieldName] - if v ~= nil then - table.insert(items, '"' .. fieldName:gsub('"', '\\"') .. '": ' .. encodeJson(v, fieldName)) - end - end - - -- Phase 2: Process remaining fields - for k, v in pairs(obj) do - local found = false - for _, fieldName in ipairs(fieldOrder) do - if k == fieldName then - found = true - break - end - end - if not found then - local keyStr = type(k) == "string" and k or tostring(k) - table.insert(items, '"' .. keyStr:gsub('"', '\\"') .. '": ' .. encodeJson(v, keyStr)) - end - end - - return "{" .. table.concat(items, ", ") .. "}" - end - else - return '"' .. tostring(obj) .. '"' - end -end - --- Helper function to set nested fields (from AWS CloudTrail) -function setNestedField(obj, path, value) - if value == nil or path == nil or path == '' then return end - local keys = {} - for key in string.gmatch(path, '[^.]+') do - if key and key ~= '' then - table.insert(keys, key) - end - end - if #keys == 0 then return end - - local current = obj - for i = 1, #keys - 1 do - local key = keys[i] - if current[key] == nil then - current[key] = {} - end - current = current[key] - end - current[keys[#keys]] = value -end - -local IGNORE_FIELDS = { - _wiz_event_type = true, - _wiz_query_time = true, - _ts = true, - _ob = true, - host = true, - source_type = true, - timestamp = true, - type = true -} - -local fieldMappings = { - -- Finding and Analytic mappings - {type="multi", source="sourceRule.description", targets={"finding.desc", "analytic.desc"}}, - {type="multi", source="sourceRule.name", targets={"finding.title", "analytic.name"}}, - {type="direct", source="sourceRule.resolutionRecommendation", target="finding.remediation.desc"}, - {type="direct", source="sourceRule.id", target="analytic.uid"}, - {type="direct", source="analytic_type_id", target="analytic.type_id"}, - {type="direct", source="analytic_type", target="analytic.type"}, - {type="direct", source="createdAt", target="finding.created_time"}, - {type="direct", source="updatedAt", target="finding.modified_time"}, - - -- Cloud mappings - {type="direct", priority1="entitySnapshot.subscription.cloudProvider", priority2="entitySnapshot.cloudPlatform", target="cloud.provider"}, - {type="direct", source="cloud_account_type", target="cloud.account.type"}, - {type="direct", source="cloud_account_type_id", target="cloud.account.type_id"}, - {type="direct", source="entitySnapshot.region", target="cloud.region"}, - {type="multi", priority1="entitySnapshot.subscription.externalId", priority2="entitySnapshot.subscriptionExternalId", targets={"cloud.account.uid", "cloud.org.uid"}}, - - -- Resource mappings - {type="direct", source="entitySnapshot.cloudProviderURL", target="resource.url"}, - {type="direct", source="entitySnapshot.externalId", target="resource.data.external_id"}, - {type="direct", source="entitySnapshot.wizResourceID.id", target="resource.data.wiz_resource_id"}, - {type="direct", source="entitySnapshot.name", target="resource.name"}, - {type="direct", source="entitySnapshot.nativeType", target="resource.type"}, - {type="direct", source="entitySnapshot.providerId", target="resource.uid"}, - {type="direct", source="entitySnapshot.resourceGroupExternalId", target="resource.group.uid"}, - {type="direct", source="entitySnapshot.status", target="resource.status"}, - {type="direct", source="entitySnapshot.type", target="resource.data.generic_type"}, - - -- Metadata and event mappings - {type="multi", source="id", targets={"metadata.uid", "finding.uid"}}, - {type="direct", source="event", target="event.type"}, - {type="direct", source="severity", target="unmapped.severity"}, - {type="multi", source="status", targets={"state", "status"}}, - {type="multi", source="datasource.vendor", targets={"dataSource.vendor", "metadata.product.vendor_name"}}, - {type="multi", source="datasource.name", targets={"dataSource.name", "metadata.product.name"}}, - {type="direct", source="datasource.category", target="dataSource.category"}, - {type="direct", source="ocsf_version", target="metadata.version"}, - - -- OCSF standard fields (computed values) - {type="computed", value="2001", target="class_uid"}, - {type="computed", value="99", target="activity_id"}, - {type="computed", value="200199", target="type_uid"}, - {type="computed", value="2", target="category_uid"}, - {type="computed", value="Security Finding", target="class_name"}, - {type="computed", value="Findings", target="category_name"}, - - -- Additional required fields - - -- Computed mappings using maps - {type="computed_map", source="severity", map="WIZ_SEVERITY_MAP", default=99, target="severity_id"}, - {type="computed_map", source="status", map="WIZ_STATE_MAP", default=99, target="state_id"}, - - -- Cloud account type mappings - {type="computed_cloud_type", source="entitySnapshot.cloudPlatform", target="cloud.account.type_id"}, - {type="computed_cloud_type", source="entitySnapshot.cloudPlatform", target="cloud.account.type"}, - -} - - --- Field ordering is now defined by the mapping order in fieldMappings table - -function getNestedField(obj, path) - if not obj or not path or path == '' then return nil end - local current = obj - for key in string.gmatch(path, '[^.]+') do - if not current or not key then return nil end - current = current[key] - end - return current -end - -function processEvent(event) - if not event then return event end - - -- Populate event.event from _wiz_event_type if it's an issue event - if event._wiz_event_type and string.lower(event._wiz_event_type) == "issue" then - event.event = "Issues" - end - - -- Check if this is an issue event (case-insensitive) - local isIssueEvent = false - if event.event and string.lower(event.event) == "issues" then - isIssueEvent = true - end - - if not isIssueEvent then - -- Return as-is for non-issue events - return event - end - - -- Add datasource field to event - if not event.datasource then - event.datasource = { - category = "security", - name = "Wiz", - vendor = "Wiz" - } - end - - -- Set event field - event.event = "Issues" - - -- Add OCSF fields - event.class_uid = "2001" - event.activity_id = "99" - event.type_uid = "200199" - event.category_uid = "2" - event.class_name = "Security Finding" - event.category_name = "Findings" - event.analytic_type = "Rule" - event.analytic_type_id = 1 - - -- Add computed fields - event.severity_id = WIZ_SEVERITY_MAP[event.severity] or 99 - event.state_id = WIZ_STATE_MAP[event.status] or 99 - - - -- Add cloud account type_id (always show) and type (only if cloudPlatform exists) - local cloudPlatform = nil - if event.entitySnapshot and event.entitySnapshot.cloudPlatform then - cloudPlatform = event.entitySnapshot.cloudPlatform - end - - event.cloud_account_type_id = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99 - - -- Check for AWS IAM users/roles - if event.cloud_account_type_id == 10 and event.entitySnapshot and event.entitySnapshot.nativeType then - local nativeType = event.entitySnapshot.nativeType - if nativeType == "user" then - event.cloud_account_type_id = 3 -- AWS IAM User - elseif nativeType == "role" then - event.cloud_account_type_id = 4 -- AWS IAM Role - end - end - - -- Only set cloud_account_type if cloudPlatform exists - if cloudPlatform then - event.cloud_account_type = OCSF_ACCOUNT_TYPE_NAME[event.cloud_account_type_id] or cloudPlatform or "Unknown Account Type" - end - -- If cloudPlatform is nil, don't set cloud_account_type but type_id still shows - - -- Add wizResourceID to entitySnapshot - if event.entitySnapshot and event.entitySnapshot.id then - event.entitySnapshot.wizResourceID = { - id = event.entitySnapshot.id - } - end - - -- Convert timestamps to Unix milliseconds - if event.createdAt then - -- Convert createdAt to Unix milliseconds - local createdAtStr = tostring(event.createdAt) - -- Try to parse as ISO timestamp and convert to milliseconds - local success, result = pcall(function() - -- Simple conversion for ISO timestamps like "2025-10-08T22:12:42.66996Z" - if createdAtStr:match("^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d") then - -- Extract the timestamp part and convert - local year, month, day, hour, min, sec = createdAtStr:match("^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)") - if year and month and day and hour and min and sec then - -- Create a simple timestamp (this is a basic implementation) - -- In a real implementation, you'd use proper date parsing - local timestamp = os.time({ - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec) - }) - return timestamp * 1000 -- Convert to milliseconds - end - end - -- Fallback: try to convert as number - local num = tonumber(createdAtStr) - if num then - -- If it's already a timestamp, ensure it's in milliseconds - if num > 1000000000000 then -- Already in milliseconds - return num - elseif num > 1000000000 then -- In seconds, convert to milliseconds - return num * 1000 - else -- Assume it's already in the right format - return num - end - end - return nil - end) - - if success and result then - event.createdAt = result - end - end - - if event.updatedAt then - -- Convert updatedAt to Unix milliseconds - local updatedAtStr = tostring(event.updatedAt) - local success, result = pcall(function() - -- Simple conversion for ISO timestamps - if updatedAtStr:match("^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d") then - local year, month, day, hour, min, sec = updatedAtStr:match("^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)") - if year and month and day and hour and min and sec then - local timestamp = os.time({ - year = tonumber(year), - month = tonumber(month), - day = tonumber(day), - hour = tonumber(hour), - min = tonumber(min), - sec = tonumber(sec) - }) - return timestamp * 1000 - end - end - local num = tonumber(updatedAtStr) - if num then - if num > 1000000000000 then - return num - elseif num > 1000000000 then - return num * 1000 - else - return num - end - end - return nil - end) - - if success and result then - event.updatedAt = result - end - end - - -- Set OCSF version - event.ocsf_version = OCSF_VERSION - - -- Remove tags field from entitySnapshot - if event.entitySnapshot then - event.entitySnapshot.tags = nil - end - - -- Track all fields that were added to the event as mapped - -- so they don't appear in unmapped - local mappedFields = {} - mappedFields["class_uid"] = true - mappedFields["activity_id"] = true - mappedFields["type_uid"] = true - mappedFields["category_uid"] = true - mappedFields["class_name"] = true - mappedFields["category_name"] = true - mappedFields["analytic_type"] = true - mappedFields["analytic_type_id"] = true - mappedFields["severity_id"] = true - mappedFields["state_id"] = true - mappedFields["cloud_account_type_id"] = true - mappedFields["cloud_account_type"] = true - mappedFields["ocsf_version"] = true - mappedFields["event"] = true - mappedFields["entitySnapshot"] = true - - local result = {} - - -- Track field order as they're processed - local fieldOrder = {} - local processedFields = {} - - -- Helper functions - local function getValue(priority1, priority2, priority3) - local value = getNestedField(event, priority1) - if value then return value end - if priority2 then - value = getNestedField(event, priority2) - if value then return value end - end - if priority3 then - return getNestedField(event, priority3) - end - return nil - end - - local function setValue(targetPath, value) - if value == "NULL_PLACEHOLDER" or value == "-" or value == "" then - setNestedField(result, targetPath, nil) - elseif value ~= nil then - setNestedField(result, targetPath, value) - end - end - - local function setMultiValue(targetPaths, value) - if value == "NULL_PLACEHOLDER" or value == "-" or value == "" then - for _, targetPath in ipairs(targetPaths) do - setNestedField(result, targetPath, nil) - end - elseif value then - for _, targetPath in ipairs(targetPaths) do - setNestedField(result, targetPath, value) - end - end - end - - -- Track field order as they're processed - local fieldOrder = {} - local processedFields = {} - - -- Apply mappings - for _, mapping in ipairs(fieldMappings) do - local value = nil - - if mapping.type == "computed" then - value = mapping.value - elseif mapping.type == "computed_map" then - -- Computed value from source using map - local sourceValue = getNestedField(event, mapping.source) - local mapTable = nil - if mapping.map == "WIZ_SEVERITY_MAP" then - mapTable = WIZ_SEVERITY_MAP - elseif mapping.map == "WIZ_STATE_MAP" then - mapTable = WIZ_STATE_MAP - end - value = (mapTable and mapTable[sourceValue]) or mapping.default - elseif mapping.type == "computed_cloud_type" then - -- Handle cloud account type logic with AWS IAM support - local cloudPlatform = getNestedField(event, mapping.source) - local typeId = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99 - - -- Check for AWS IAM users/roles - if typeId == 10 and getNestedField(event, "entitySnapshot.nativeType") then - local nativeType = getNestedField(event, "entitySnapshot.nativeType") - if nativeType == "user" then - typeId = 3 -- AWS IAM User - elseif nativeType == "role" then - typeId = 4 -- AWS IAM Role - end - end - - if mapping.target == "cloud.account.type_id" or mapping.target == "message.cloud_account_type_id" then - value = typeId -- Always show type_id - elseif mapping.target == "cloud.account.type" or mapping.target == "message.cloud_account_type" then - -- Only show type if cloudPlatform exists - if cloudPlatform then - value = OCSF_ACCOUNT_TYPE_NAME[typeId] or cloudPlatform or "Unknown Account Type" - end - -- If cloudPlatform is nil, value remains nil (field will be ignored) - end - elseif mapping.type == "message_field" then - value = getNestedField(event, mapping.source) - elseif mapping.priority1 and mapping.priority2 then - value = getValue(mapping.priority1, mapping.priority2, mapping.priority3) - else - value = getNestedField(event, mapping.source) - end - - -- Set the value and track order - if mapping.type == "direct" then - setValue(mapping.target, value) - if not processedFields[mapping.target] then - table.insert(fieldOrder, mapping.target) - processedFields[mapping.target] = true - end - elseif mapping.type == "multi" then - setMultiValue(mapping.targets, value) - for _, target in ipairs(mapping.targets) do - if not processedFields[target] then - table.insert(fieldOrder, target) - processedFields[target] = true - end - end - elseif mapping.type == "computed" or mapping.type == "computed_map" then - setValue(mapping.target, value) - if not processedFields[mapping.target] then - table.insert(fieldOrder, mapping.target) - processedFields[mapping.target] = true - end - elseif mapping.type == "computed_cloud_type" then - if mapping.target:match("^message%.") then - -- Handle message fields separately - if not result["message"] then - result["message"] = {} - end - setNestedField(result["message"], mapping.target:gsub("message%.", ""), value) - else - setValue(mapping.target, value) - if not processedFields[mapping.target] then - table.insert(fieldOrder, mapping.target) - processedFields[mapping.target] = true - end - end - end - - -- Track mapped fields for unmapped processing - if mapping.source then - mappedFields[mapping.source] = true - end - if mapping.priority1 then - mappedFields[mapping.priority1] = true - end - if mapping.priority2 then - mappedFields[mapping.priority2] = true - end - if mapping.priority3 then - mappedFields[mapping.priority3] = true - end - - -- For computed fields, we need to track the target fields as mapped - -- since they will be added to the event and shouldn't appear in unmapped - if mapping.type == "computed" or mapping.type == "computed_map" or mapping.type == "computed_cloud_type" then - if mapping.target then - -- Extract the root field name from the target path - local rootField = mapping.target:match("^([^%.]+)") - if rootField then - mappedFields[rootField] = true - end - end - end - end - - -- Only include fields that are NOT mapped and NOT ignored - for key, value in pairs(event) do - if not IGNORE_FIELDS[key] and not mappedFields[key] then - if type(value) == 'table' then - local nestedObj = {} - for nestedKey, nestedValue in pairs(value) do - if type(nestedValue) ~= 'function' and nestedValue ~= "-" and nestedValue ~= "" then - -- Check if this nested field was mapped - local nestedFieldKey = key .. "." .. nestedKey - if not mappedFields[nestedFieldKey] then - nestedObj[nestedKey] = nestedValue - end - end - end - if next(nestedObj) then - result["unmapped." .. key] = nestedObj - end - elseif value ~= "-" and value ~= "" then - result["unmapped." .. key] = value - end - end - end - - -- Build nested structure instead of flattening - local nested = {} - - -- Add mapped fields to nested structure - for _, fieldName in ipairs(fieldOrder) do - local value = getNestedField(result, fieldName) - if value ~= nil then - setNestedField(nested, fieldName, value) - end - end - - -- Add message field as JSON string (like AWS CloudTrail does) - local cleanEvent = {} - for key, value in pairs(event) do - if key ~= "_ob" and key ~= "timestamp" and key ~= "_ts" and key ~= "_wiz_event_type" and key ~= "_wiz_query_time" and value ~= "-" and value ~= "" then - cleanEvent[key] = value - end - end - - nested["message"] = encodeJson(cleanEvent, "message") - - -- Add missing fields that should be in the message - if cleanEvent.readOnly == nil then - cleanEvent.readOnly = false -- Default for Wiz events - end - if cleanEvent.eventType == nil then - cleanEvent.eventType = "Issues" -- Default for Wiz events - end - - -- Add unmapped fields as nested object (using setNestedField like AWS CloudTrail) - for key, value in pairs(result) do - if key:match("^unmapped%.") then - local fieldName = key:gsub("^unmapped%.", "") - setNestedField(nested, "unmapped." .. fieldName, value) - end - end - - -- Add time field - if nested.finding and nested.finding.modified_time then - nested["time"] = nested.finding.modified_time - end - - if FEATURES.FLATTEN_EVENT_TYPE then - if nested and nested.event then - nested['event.type'] = nested.event.type - end - end - return nested -end - diff --git a/pipelines/community/transform_ocsf/wiz_issue/wiz_issue.json b/pipelines/community/transform_ocsf/wiz_issue/wiz_issue.json deleted file mode 100644 index 213162b..0000000 --- a/pipelines/community/transform_ocsf/wiz_issue/wiz_issue.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "schema_version": "1.0", - "component": "transform", - "template_name": "OCSFSerializer", - "display_name": "Wiz Issue", - "grade": { - "letter": "D", - "score": 60, - "verdict": "analyzer_limit", - "required_field_coverage_pct": 0, - "source_field_coverage_pct": 100, - "tested_with_realistic_event": true, - "validated_at": "2026-04-19" - }, - "ocsf": { - "class_uid": null, - "class_name": null, - "category_uid": null, - "category_name": null, - "version": "1.3.0" - }, - "description": "OCSF-compliant Lua serializer for Wiz Issue. Maps source events to OCSF (unclassified) class_uid n/a.", - "vendor": "wiz", - "source_name": "wiz_issue", - "version": "1.0.0-rc.3", - "parameters": [ - { - "name": "serializer", - "type": "select", - "value": "wiz-issue-lua", - "description": "Serializer identifier used by Observo runtime" - }, - { - "name": "lua", - "type": "lua_code", - "value": "\nlocal FEATURES = {\n FLATTEN_EVENT_TYPE = true,\n}\n\n-- Wiz to OCSF Mapping Script\nlocal OCSF_VERSION = \"v1.0.0-rc.3\"\n\nlocal WIZ_SEVERITY_MAP = {[\"INFORMATIONAL\"]=1, [\"LOW\"]=2, [\"MEDIUM\"]=3, [\"HIGH\"]=4, [\"CRITICAL\"]=5, [\"OTHER\"]=99}\nlocal WIZ_STATE_MAP = {[\"OPEN\"]=1, [\"IN_PROGRESS\"]=2, [\"REJECTED\"]=3, [\"RESOLVED\"]=4, [\"OTHER\"]=99}\nlocal OCSF_ACCOUNT_TYPE_ID = {[\"AWS\"]=10, [\"GCP\"]=5, [\"EKS\"]=10, [\"GKE\"]=5}\nlocal OCSF_ACCOUNT_TYPE_NAME = {[10]=\"AWS Account\", [5]=\"GCP Account\", [3]=\"AWS IAM User\", [4]=\"AWS IAM Role\"}\n\n-- Field ordering templates for consistent JSON serialization\nlocal FIELD_ORDERS = {\n root = {\"activity_id\", \"analytic\", \"category_name\", \"category_uid\", \"class_name\", \"class_uid\",\n \"cloud\", \"dataSource\", \"event\", \"finding\", \"index\", \"metadata\", \"message\", \"resource\",\n \"severity\", \"severity_id\", \"state\", \"state_id\",\n \"status\", \"type_uid\", \"unmapped\"\n },\n message = {\n \"activity_id\", \"analytic_type\", \"analytic_type_id\", \"category_name\", \"category_uid\",\n \"class_name\", \"class_uid\", \"cloud_account_type\", \"cloud_account_type_id\", \"datasource\",\n \"entitySnapshot\", \"event\", \"id\", \"ocsf_version\", \"severity\", \"severity_id\",\n \"sourceRule\", \"state_id\", \"status\", \"type_uid\", \"updatedAt\", \"eventType\", \"readOnly\", \"createdAt\"\n },\n entitySnapshot = {\n \"cloudPlatform\", \"cloudProviderURL\", \"externalId\", \"id\", \"name\", \"nativeType\",\n \"projects\", \"providerId\", \"region\", \"resourceGroupExternalId\", \"status\",\n \"subscription\", \"subscriptionExternalId\", \"type\", \"wizResourceID\"\n },\n sourceRule = {\n \"description\", \"id\", \"name\", \"remediationInstructions\", \"resolutionRecommendation\"\n },\n datasource = {\n \"category\", \"name\", \"vendor\"\n },\n subscription = {\n \"externalId\", \"name\", \"id\", \"cloudProvider\"\n },\n projects = {\n \"name\", \"riskProfile\", \"id\"\n },\n riskProfile = {\n \"businessImpact\"\n },\n unmapped = {\n \"severity\"\n }\n}\n\n-- JSON encoding function (from AWS CloudTrail)\nfunction encodeJson(obj, key)\n if obj == nil or obj == \"NULL_PLACEHOLDER\" then\n return \"null\"\n elseif type(obj) == \"boolean\" then\n return tostring(obj)\n elseif type(obj) == \"number\" then\n return tostring(obj)\n elseif type(obj) == \"string\" then\n return '\"' .. obj:gsub('\"', '\\\\\"') .. '\"'\n elseif type(obj) == \"table\" then\n local isArray = true\n local maxIndex = 0\n for k, v in pairs(obj) do\n if type(k) ~= \"number\" then\n isArray = false\n break\n end\n maxIndex = math.max(maxIndex, k)\n end\n \n if isArray and maxIndex > 0 then\n local items = {}\n for i = 1, maxIndex do\n local elementKey = key or tostring(i)\n table.insert(items, obj[i] ~= nil and encodeJson(obj[i], elementKey) or \"null\")\n end\n return \"[\" .. table.concat(items, \", \") .. \"]\"\n else\n local items = {}\n local fieldOrder = FIELD_ORDERS[key] or {}\n \n -- Phase 1: Process fields in predefined order\n for _, fieldName in ipairs(fieldOrder) do\n local v = obj[fieldName]\n if v ~= nil then\n table.insert(items, '\"' .. fieldName:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, fieldName))\n end\n end\n \n -- Phase 2: Process remaining fields\n for k, v in pairs(obj) do\n local found = false\n for _, fieldName in ipairs(fieldOrder) do\n if k == fieldName then \n found = true\n break\n end\n end\n if not found then\n local keyStr = type(k) == \"string\" and k or tostring(k)\n table.insert(items, '\"' .. keyStr:gsub('\"', '\\\\\"') .. '\": ' .. encodeJson(v, keyStr))\n end\n end\n \n return \"{\" .. table.concat(items, \", \") .. \"}\"\n end\n else\n return '\"' .. tostring(obj) .. '\"'\n end\nend\n\n-- Helper function to set nested fields (from AWS CloudTrail)\nfunction setNestedField(obj, path, value)\n if value == nil or path == nil or path == '' then return end\n local keys = {}\n for key in string.gmatch(path, '[^.]+') do\n if key and key ~= '' then\n table.insert(keys, key)\n end\n end\n if #keys == 0 then return end\n \n local current = obj\n for i = 1, #keys - 1 do\n local key = keys[i]\n if current[key] == nil then\n current[key] = {}\n end\n current = current[key]\n end\n current[keys[#keys]] = value\nend\n\nlocal IGNORE_FIELDS = { \n _wiz_event_type = true, \n _wiz_query_time = true, \n _ts = true, \n _ob = true,\n host = true,\n source_type = true,\n timestamp = true,\n type = true\n}\n\nlocal fieldMappings = {\n -- Finding and Analytic mappings\n {type=\"multi\", source=\"sourceRule.description\", targets={\"finding.desc\", \"analytic.desc\"}},\n {type=\"multi\", source=\"sourceRule.name\", targets={\"finding.title\", \"analytic.name\"}},\n {type=\"direct\", source=\"sourceRule.resolutionRecommendation\", target=\"finding.remediation.desc\"},\n {type=\"direct\", source=\"sourceRule.id\", target=\"analytic.uid\"},\n {type=\"direct\", source=\"analytic_type_id\", target=\"analytic.type_id\"},\n {type=\"direct\", source=\"analytic_type\", target=\"analytic.type\"},\n {type=\"direct\", source=\"createdAt\", target=\"finding.created_time\"},\n {type=\"direct\", source=\"updatedAt\", target=\"finding.modified_time\"},\n \n -- Cloud mappings\n {type=\"direct\", priority1=\"entitySnapshot.subscription.cloudProvider\", priority2=\"entitySnapshot.cloudPlatform\", target=\"cloud.provider\"},\n {type=\"direct\", source=\"cloud_account_type\", target=\"cloud.account.type\"},\n {type=\"direct\", source=\"cloud_account_type_id\", target=\"cloud.account.type_id\"},\n {type=\"direct\", source=\"entitySnapshot.region\", target=\"cloud.region\"},\n {type=\"multi\", priority1=\"entitySnapshot.subscription.externalId\", priority2=\"entitySnapshot.subscriptionExternalId\", targets={\"cloud.account.uid\", \"cloud.org.uid\"}},\n \n -- Resource mappings\n {type=\"direct\", source=\"entitySnapshot.cloudProviderURL\", target=\"resource.url\"},\n {type=\"direct\", source=\"entitySnapshot.externalId\", target=\"resource.data.external_id\"},\n {type=\"direct\", source=\"entitySnapshot.wizResourceID.id\", target=\"resource.data.wiz_resource_id\"},\n {type=\"direct\", source=\"entitySnapshot.name\", target=\"resource.name\"},\n {type=\"direct\", source=\"entitySnapshot.nativeType\", target=\"resource.type\"},\n {type=\"direct\", source=\"entitySnapshot.providerId\", target=\"resource.uid\"},\n {type=\"direct\", source=\"entitySnapshot.resourceGroupExternalId\", target=\"resource.group.uid\"},\n {type=\"direct\", source=\"entitySnapshot.status\", target=\"resource.status\"},\n {type=\"direct\", source=\"entitySnapshot.type\", target=\"resource.data.generic_type\"},\n \n -- Metadata and event mappings\n {type=\"multi\", source=\"id\", targets={\"metadata.uid\", \"finding.uid\"}},\n {type=\"direct\", source=\"event\", target=\"event.type\"},\n {type=\"direct\", source=\"severity\", target=\"unmapped.severity\"},\n {type=\"multi\", source=\"status\", targets={\"state\", \"status\"}},\n {type=\"multi\", source=\"datasource.vendor\", targets={\"dataSource.vendor\", \"metadata.product.vendor_name\"}},\n {type=\"multi\", source=\"datasource.name\", targets={\"dataSource.name\", \"metadata.product.name\"}},\n {type=\"direct\", source=\"datasource.category\", target=\"dataSource.category\"},\n {type=\"direct\", source=\"ocsf_version\", target=\"metadata.version\"},\n \n -- OCSF standard fields (computed values)\n {type=\"computed\", value=\"2001\", target=\"class_uid\"},\n {type=\"computed\", value=\"99\", target=\"activity_id\"},\n {type=\"computed\", value=\"200199\", target=\"type_uid\"},\n {type=\"computed\", value=\"2\", target=\"category_uid\"},\n {type=\"computed\", value=\"Security Finding\", target=\"class_name\"},\n {type=\"computed\", value=\"Findings\", target=\"category_name\"},\n \n -- Additional required fields\n \n -- Computed mappings using maps\n {type=\"computed_map\", source=\"severity\", map=\"WIZ_SEVERITY_MAP\", default=99, target=\"severity_id\"},\n {type=\"computed_map\", source=\"status\", map=\"WIZ_STATE_MAP\", default=99, target=\"state_id\"},\n \n -- Cloud account type mappings\n {type=\"computed_cloud_type\", source=\"entitySnapshot.cloudPlatform\", target=\"cloud.account.type_id\"},\n {type=\"computed_cloud_type\", source=\"entitySnapshot.cloudPlatform\", target=\"cloud.account.type\"},\n \n}\n\n\n-- Field ordering is now defined by the mapping order in fieldMappings table\n\nfunction getNestedField(obj, path)\n if not obj or not path or path == '' then return nil end\n local current = obj\n for key in string.gmatch(path, '[^.]+') do\n if not current or not key then return nil end\n current = current[key]\n end\n return current\nend\n\nfunction processEvent(event)\n if not event then return event end\n \n -- Populate event.event from _wiz_event_type if it's an issue event\n if event._wiz_event_type and string.lower(event._wiz_event_type) == \"issue\" then\n event.event = \"Issues\"\n end\n \n -- Check if this is an issue event (case-insensitive)\n local isIssueEvent = false\n if event.event and string.lower(event.event) == \"issues\" then\n isIssueEvent = true\n end\n \n if not isIssueEvent then\n -- Return as-is for non-issue events\n return event\n end\n\n -- Add datasource field to event\n if not event.datasource then\n event.datasource = {\n category = \"security\",\n name = \"Wiz\",\n vendor = \"Wiz\"\n }\n end\n \n -- Set event field\n event.event = \"Issues\"\n \n -- Add OCSF fields\n event.class_uid = \"2001\"\n event.activity_id = \"99\"\n event.type_uid = \"200199\"\n event.category_uid = \"2\"\n event.class_name = \"Security Finding\"\n event.category_name = \"Findings\"\n event.analytic_type = \"Rule\"\n event.analytic_type_id = 1\n \n -- Add computed fields \n event.severity_id = WIZ_SEVERITY_MAP[event.severity] or 99\n event.state_id = WIZ_STATE_MAP[event.status] or 99\n \n \n -- Add cloud account type_id (always show) and type (only if cloudPlatform exists)\n local cloudPlatform = nil\n if event.entitySnapshot and event.entitySnapshot.cloudPlatform then\n cloudPlatform = event.entitySnapshot.cloudPlatform\n end\n \n event.cloud_account_type_id = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99\n \n -- Check for AWS IAM users/roles \n if event.cloud_account_type_id == 10 and event.entitySnapshot and event.entitySnapshot.nativeType then\n local nativeType = event.entitySnapshot.nativeType\n if nativeType == \"user\" then\n event.cloud_account_type_id = 3 -- AWS IAM User\n elseif nativeType == \"role\" then\n event.cloud_account_type_id = 4 -- AWS IAM Role\n end\n end\n \n -- Only set cloud_account_type if cloudPlatform exists\n if cloudPlatform then\n event.cloud_account_type = OCSF_ACCOUNT_TYPE_NAME[event.cloud_account_type_id] or cloudPlatform or \"Unknown Account Type\"\n end\n -- If cloudPlatform is nil, don't set cloud_account_type but type_id still shows\n \n -- Add wizResourceID to entitySnapshot \n if event.entitySnapshot and event.entitySnapshot.id then\n event.entitySnapshot.wizResourceID = {\n id = event.entitySnapshot.id\n }\n end\n \n -- Convert timestamps to Unix milliseconds \n if event.createdAt then\n -- Convert createdAt to Unix milliseconds\n local createdAtStr = tostring(event.createdAt)\n -- Try to parse as ISO timestamp and convert to milliseconds\n local success, result = pcall(function()\n -- Simple conversion for ISO timestamps like \"2025-10-08T22:12:42.66996Z\"\n if createdAtStr:match(\"^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d\") then\n -- Extract the timestamp part and convert\n local year, month, day, hour, min, sec = createdAtStr:match(\"^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)\")\n if year and month and day and hour and min and sec then\n -- Create a simple timestamp (this is a basic implementation)\n -- In a real implementation, you'd use proper date parsing\n local timestamp = os.time({\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec)\n })\n return timestamp * 1000 -- Convert to milliseconds\n end\n end\n -- Fallback: try to convert as number\n local num = tonumber(createdAtStr)\n if num then\n -- If it's already a timestamp, ensure it's in milliseconds\n if num > 1000000000000 then -- Already in milliseconds\n return num\n elseif num > 1000000000 then -- In seconds, convert to milliseconds\n return num * 1000\n else -- Assume it's already in the right format\n return num\n end\n end\n return nil\n end)\n \n if success and result then\n event.createdAt = result\n end\n end\n \n if event.updatedAt then\n -- Convert updatedAt to Unix milliseconds\n local updatedAtStr = tostring(event.updatedAt)\n local success, result = pcall(function()\n -- Simple conversion for ISO timestamps\n if updatedAtStr:match(\"^%d%d%d%d%-%d%d%-%d%dT%d%d:%d%d:%d%d\") then\n local year, month, day, hour, min, sec = updatedAtStr:match(\"^(%d%d%d%d)%-(%d%d)%-(%d%d)T(%d%d):(%d%d):(%d%d)\")\n if year and month and day and hour and min and sec then\n local timestamp = os.time({\n year = tonumber(year),\n month = tonumber(month),\n day = tonumber(day),\n hour = tonumber(hour),\n min = tonumber(min),\n sec = tonumber(sec)\n })\n return timestamp * 1000\n end\n end\n local num = tonumber(updatedAtStr)\n if num then\n if num > 1000000000000 then\n return num\n elseif num > 1000000000 then\n return num * 1000\n else\n return num\n end\n end\n return nil\n end)\n \n if success and result then\n event.updatedAt = result\n end\n end\n \n -- Set OCSF version \n event.ocsf_version = OCSF_VERSION\n \n -- Remove tags field from entitySnapshot \n if event.entitySnapshot then\n event.entitySnapshot.tags = nil\n end\n\n -- Track all fields that were added to the event as mapped\n -- so they don't appear in unmapped\n local mappedFields = {}\n mappedFields[\"class_uid\"] = true\n mappedFields[\"activity_id\"] = true\n mappedFields[\"type_uid\"] = true\n mappedFields[\"category_uid\"] = true\n mappedFields[\"class_name\"] = true\n mappedFields[\"category_name\"] = true\n mappedFields[\"analytic_type\"] = true\n mappedFields[\"analytic_type_id\"] = true\n mappedFields[\"severity_id\"] = true\n mappedFields[\"state_id\"] = true\n mappedFields[\"cloud_account_type_id\"] = true\n mappedFields[\"cloud_account_type\"] = true\n mappedFields[\"ocsf_version\"] = true\n mappedFields[\"event\"] = true\n mappedFields[\"entitySnapshot\"] = true\n\n local result = {}\n \n -- Track field order as they're processed\n local fieldOrder = {}\n local processedFields = {}\n \n -- Helper functions\n local function getValue(priority1, priority2, priority3)\n local value = getNestedField(event, priority1)\n if value then return value end\n if priority2 then\n value = getNestedField(event, priority2)\n if value then return value end\n end\n if priority3 then\n return getNestedField(event, priority3)\n end\n return nil\n end\n \n local function setValue(targetPath, value)\n if value == \"NULL_PLACEHOLDER\" or value == \"-\" or value == \"\" then\n setNestedField(result, targetPath, nil)\n elseif value ~= nil then\n setNestedField(result, targetPath, value)\n end\n end\n \n local function setMultiValue(targetPaths, value)\n if value == \"NULL_PLACEHOLDER\" or value == \"-\" or value == \"\" then\n for _, targetPath in ipairs(targetPaths) do\n setNestedField(result, targetPath, nil)\n end\n elseif value then\n for _, targetPath in ipairs(targetPaths) do\n setNestedField(result, targetPath, value)\n end\n end\n end\n \n -- Track field order as they're processed\n local fieldOrder = {}\n local processedFields = {}\n \n -- Apply mappings\n for _, mapping in ipairs(fieldMappings) do\n local value = nil\n \n if mapping.type == \"computed\" then\n value = mapping.value\n elseif mapping.type == \"computed_map\" then\n -- Computed value from source using map\n local sourceValue = getNestedField(event, mapping.source)\n local mapTable = nil\n if mapping.map == \"WIZ_SEVERITY_MAP\" then\n mapTable = WIZ_SEVERITY_MAP\n elseif mapping.map == \"WIZ_STATE_MAP\" then\n mapTable = WIZ_STATE_MAP\n end\n value = (mapTable and mapTable[sourceValue]) or mapping.default\n elseif mapping.type == \"computed_cloud_type\" then\n -- Handle cloud account type logic with AWS IAM support\n local cloudPlatform = getNestedField(event, mapping.source)\n local typeId = OCSF_ACCOUNT_TYPE_ID[cloudPlatform] or 99\n \n -- Check for AWS IAM users/roles \n if typeId == 10 and getNestedField(event, \"entitySnapshot.nativeType\") then\n local nativeType = getNestedField(event, \"entitySnapshot.nativeType\")\n if nativeType == \"user\" then\n typeId = 3 -- AWS IAM User\n elseif nativeType == \"role\" then\n typeId = 4 -- AWS IAM Role\n end\n end\n \n if mapping.target == \"cloud.account.type_id\" or mapping.target == \"message.cloud_account_type_id\" then\n value = typeId -- Always show type_id\n elseif mapping.target == \"cloud.account.type\" or mapping.target == \"message.cloud_account_type\" then\n -- Only show type if cloudPlatform exists\n if cloudPlatform then\n value = OCSF_ACCOUNT_TYPE_NAME[typeId] or cloudPlatform or \"Unknown Account Type\"\n end\n -- If cloudPlatform is nil, value remains nil (field will be ignored)\n end\n elseif mapping.type == \"message_field\" then\n value = getNestedField(event, mapping.source)\n elseif mapping.priority1 and mapping.priority2 then\n value = getValue(mapping.priority1, mapping.priority2, mapping.priority3)\n else\n value = getNestedField(event, mapping.source)\n end\n \n -- Set the value and track order\n if mapping.type == \"direct\" then\n setValue(mapping.target, value)\n if not processedFields[mapping.target] then\n table.insert(fieldOrder, mapping.target)\n processedFields[mapping.target] = true\n end\n elseif mapping.type == \"multi\" then\n setMultiValue(mapping.targets, value)\n for _, target in ipairs(mapping.targets) do\n if not processedFields[target] then\n table.insert(fieldOrder, target)\n processedFields[target] = true\n end\n end\n elseif mapping.type == \"computed\" or mapping.type == \"computed_map\" then\n setValue(mapping.target, value)\n if not processedFields[mapping.target] then\n table.insert(fieldOrder, mapping.target)\n processedFields[mapping.target] = true\n end\n elseif mapping.type == \"computed_cloud_type\" then\n if mapping.target:match(\"^message%.\") then\n -- Handle message fields separately\n if not result[\"message\"] then\n result[\"message\"] = {}\n end\n setNestedField(result[\"message\"], mapping.target:gsub(\"message%.\", \"\"), value)\n else\n setValue(mapping.target, value)\n if not processedFields[mapping.target] then\n table.insert(fieldOrder, mapping.target)\n processedFields[mapping.target] = true\n end\n end\n end\n \n -- Track mapped fields for unmapped processing\n if mapping.source then\n mappedFields[mapping.source] = true\n end\n if mapping.priority1 then\n mappedFields[mapping.priority1] = true\n end\n if mapping.priority2 then\n mappedFields[mapping.priority2] = true\n end\n if mapping.priority3 then\n mappedFields[mapping.priority3] = true\n end\n \n -- For computed fields, we need to track the target fields as mapped\n -- since they will be added to the event and shouldn't appear in unmapped\n if mapping.type == \"computed\" or mapping.type == \"computed_map\" or mapping.type == \"computed_cloud_type\" then\n if mapping.target then\n -- Extract the root field name from the target path\n local rootField = mapping.target:match(\"^([^%.]+)\")\n if rootField then\n mappedFields[rootField] = true\n end\n end\n end\n end\n \n -- Only include fields that are NOT mapped and NOT ignored\n for key, value in pairs(event) do\n if not IGNORE_FIELDS[key] and not mappedFields[key] then\n if type(value) == 'table' then\n local nestedObj = {}\n for nestedKey, nestedValue in pairs(value) do\n if type(nestedValue) ~= 'function' and nestedValue ~= \"-\" and nestedValue ~= \"\" then\n -- Check if this nested field was mapped\n local nestedFieldKey = key .. \".\" .. nestedKey\n if not mappedFields[nestedFieldKey] then\n nestedObj[nestedKey] = nestedValue\n end\n end\n end\n if next(nestedObj) then\n result[\"unmapped.\" .. key] = nestedObj\n end\n elseif value ~= \"-\" and value ~= \"\" then\n result[\"unmapped.\" .. key] = value\n end\n end\n end\n \n -- Build nested structure instead of flattening\n local nested = {}\n \n -- Add mapped fields to nested structure\n for _, fieldName in ipairs(fieldOrder) do\n local value = getNestedField(result, fieldName)\n if value ~= nil then \n setNestedField(nested, fieldName, value)\n end\n end\n \n -- Add message field as JSON string (like AWS CloudTrail does)\n local cleanEvent = {}\n for key, value in pairs(event) do\n if key ~= \"_ob\" and key ~= \"timestamp\" and key ~= \"_ts\" and key ~= \"_wiz_event_type\" and key ~= \"_wiz_query_time\" and value ~= \"-\" and value ~= \"\" then\n cleanEvent[key] = value\n end\n end\n \n nested[\"message\"] = encodeJson(cleanEvent, \"message\")\n\n -- Add missing fields that should be in the message\n if cleanEvent.readOnly == nil then\n cleanEvent.readOnly = false -- Default for Wiz events\n end\n if cleanEvent.eventType == nil then\n cleanEvent.eventType = \"Issues\" -- Default for Wiz events\n end\n \n -- Add unmapped fields as nested object (using setNestedField like AWS CloudTrail)\n for key, value in pairs(result) do\n if key:match(\"^unmapped%.\") then\n local fieldName = key:gsub(\"^unmapped%.\", \"\")\n setNestedField(nested, \"unmapped.\" .. fieldName, value)\n end\n end\n\n -- Add time field\n if nested.finding and nested.finding.modified_time then\n nested[\"time\"] = nested.finding.modified_time\n end\n \n if FEATURES.FLATTEN_EVENT_TYPE then\n if nested and nested.event then\n nested['event.type'] = nested.event.type\n end\n end\n return nested\nend\n\n", - "description": "Lua processEvent serializer \u2014 maps source fields to OCSF schema" - } - ], - "validation": { - "harness_grade": "D", - "harness_score": 60, - "harness_lint_score": 0.0, - "harness_required_coverage": 0, - "harness_field_coverage": 100, - "lua_validity": "pass", - "execution_passed": true, - "tested_with_realistic_event": true, - "orion_reviewed": true, - "orion_verdict": "analyzer_limit", - "validated_at": "2026-04-19" - }, - "provenance": { - "tier": "ui", - "source": "Observo.ai Pipeline Manager UI (production template)" - } -} \ No newline at end of file