Add support for synthetic_source_keep = none (#2422)

Add support for synthetic_source_keep mapping in generated elasticsearch component files, and add this mapping field to all ECS fields that represent sets.

synthetic_source_keep = none indicates that field is an unordered set, and helps improve storage efficiency with Elasticsearch logsdb index mode.

CHANGELOG.next.md

@@ -22,6 +22,7 @@ Thanks, you're awesome :-) -->
 
 * Define base encoding of `x509.serial_number`. #2383
 * Restrict the encoding of `x509.serial_number` to base 16. #2398
+* Set synthetic_source_keep = none on fields that represent sets. #2422
 
 #### Deprecated
 

experimental/generated/ecs/ecs_flat.yml

@@ -650,6 +650,7 @@ client.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 cloud.account.id:
   dashed_name: cloud-account-id
@@ -1187,6 +1188,7 @@ container.image.tag:
     relation: equivalent
     stability: experimental
   short: Container image tags.
+  synthetic_source_keep: none
   type: keyword
 container.labels:
   dashed_name: container-labels
@@ -1790,6 +1792,7 @@ destination.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 device.id:
   dashed_name: device-id
@@ -2506,6 +2509,7 @@ dns.header_flags:
   normalize:
   - array
   short: Array of DNS header flags.
+  synthetic_source_keep: none
   type: keyword
 dns.id:
   dashed_name: dns-id
@@ -2843,6 +2847,7 @@ email.bcc.address:
   normalize:
   - array
   short: Email address of BCC recipient
+  synthetic_source_keep: none
   type: keyword
 email.cc.address:
   dashed_name: email-cc-address
@@ -2855,6 +2860,7 @@ email.cc.address:
   normalize:
   - array
   short: Email address of CC recipient
+  synthetic_source_keep: none
   type: keyword
 email.content_type:
   dashed_name: email-content-type
@@ -2903,6 +2909,7 @@ email.from.address:
   normalize:
   - array
   short: The sender's email address.
+  synthetic_source_keep: none
   type: keyword
 email.local_id:
   dashed_name: email-local-id
@@ -2952,6 +2959,7 @@ email.reply_to.address:
   normalize:
   - array
   short: Address replies should be delivered to.
+  synthetic_source_keep: none
   type: keyword
 email.sender.address:
   dashed_name: email-sender-address
@@ -2963,6 +2971,7 @@ email.sender.address:
   name: sender.address
   normalize: []
   short: Address of the message sender.
+  synthetic_source_keep: none
   type: keyword
 email.subject:
   dashed_name: email-subject
@@ -2990,6 +2999,7 @@ email.to.address:
   normalize:
   - array
   short: Email address of recipient
+  synthetic_source_keep: none
   type: keyword
 email.x_mailer:
   dashed_name: email-x-mailer
@@ -3347,6 +3357,7 @@ event.category:
   normalize:
   - array
   short: Event category. The second categorization field in the hierarchy.
+  synthetic_source_keep: none
   type: keyword
 event.code:
   dashed_name: event-code
@@ -3908,6 +3919,7 @@ event.type:
   normalize:
   - array
   short: Event type. The third categorization field in the hierarchy.
+  synthetic_source_keep: none
   type: keyword
 event.url:
   dashed_name: event-url
@@ -4058,6 +4070,7 @@ file.attributes:
   - relation: match
     stability: experimental
   short: Array of file attributes.
+  synthetic_source_keep: none
   type: keyword
 file.code_signature.digest_algorithm:
   dashed_name: file-code-signature-digest-algorithm
@@ -6127,6 +6140,7 @@ host.ip:
   - relation: match
     stability: experimental
   short: Host ip addresses.
+  synthetic_source_keep: none
   type: ip
 host.mac:
   dashed_name: host-mac
@@ -6147,6 +6161,7 @@ host.mac:
     stability: experimental
   pattern: ^[A-F0-9]{2}(-[A-F0-9]{2}){5,}$
   short: Host MAC addresses.
+  synthetic_source_keep: none
   type: keyword
 host.name:
   dashed_name: host-name
@@ -7505,6 +7520,7 @@ observer.ip:
   normalize:
   - array
   short: IP addresses of the observer.
+  synthetic_source_keep: none
   type: ip
 observer.mac:
   dashed_name: observer-mac
@@ -7522,6 +7538,7 @@ observer.mac:
   - array
   pattern: ^[A-F0-9]{2}(-[A-F0-9]{2}){5,}$
   short: MAC addresses of the observer.
+  synthetic_source_keep: none
   type: keyword
 observer.name:
   dashed_name: observer-name
@@ -7784,6 +7801,7 @@ orchestrator.resource.annotation:
   normalize:
   - array
   short: The list of annotations added to the resource.
+  synthetic_source_keep: none
   type: keyword
 orchestrator.resource.id:
   dashed_name: orchestrator-resource-id
@@ -7806,6 +7824,7 @@ orchestrator.resource.ip:
   normalize:
   - array
   short: IP address assigned to the resource associated with the event being observed.
+  synthetic_source_keep: none
   type: ip
 orchestrator.resource.label:
   dashed_name: orchestrator-resource-label
@@ -7818,6 +7837,7 @@ orchestrator.resource.label:
   normalize:
   - array
   short: The list of labels added to the resource.
+  synthetic_source_keep: none
   type: keyword
 orchestrator.resource.name:
   dashed_name: orchestrator-resource-name
@@ -9317,6 +9337,7 @@ process.env_vars:
   normalize:
   - array
   short: Array of environment variable bindings.
+  synthetic_source_keep: none
   type: keyword
 process.executable:
   dashed_name: process-executable
@@ -11783,6 +11804,7 @@ process.parent.thread.capabilities.effective:
   original_fieldset: process
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities used for permission checks.
+  synthetic_source_keep: none
   type: keyword
 process.parent.thread.capabilities.permitted:
   dashed_name: process-parent-thread-capabilities-permitted
@@ -11798,6 +11820,7 @@ process.parent.thread.capabilities.permitted:
   original_fieldset: process
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities a thread could assume.
+  synthetic_source_keep: none
   type: keyword
 process.parent.thread.id:
   dashed_name: process-parent-thread-id
@@ -13009,6 +13032,7 @@ process.thread.capabilities.effective:
   - array
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities used for permission checks.
+  synthetic_source_keep: none
   type: keyword
 process.thread.capabilities.permitted:
   dashed_name: process-thread-capabilities-permitted
@@ -13023,6 +13047,7 @@ process.thread.capabilities.permitted:
   - array
   pattern: ^(CAP_[A-Z_]+|\d+)$
   short: Array of capabilities a thread could assume.
+  synthetic_source_keep: none
   type: keyword
 process.thread.id:
   dashed_name: process-thread-id
@@ -13314,6 +13339,7 @@ related.hash:
   otel:
   - relation: na
   short: All the hashes seen on your event.
+  synthetic_source_keep: none
   type: keyword
 related.hosts:
   dashed_name: related-hosts
@@ -13328,6 +13354,7 @@ related.hosts:
   otel:
   - relation: na
   short: All the host identifiers seen on your event.
+  synthetic_source_keep: none
   type: keyword
 related.ip:
   dashed_name: related-ip
@@ -13340,6 +13367,7 @@ related.ip:
   otel:
   - relation: na
   short: All of the IPs seen on your event.
+  synthetic_source_keep: none
   type: ip
 related.user:
   dashed_name: related-user
@@ -13353,6 +13381,7 @@ related.user:
   otel:
   - relation: na
   short: All the user names or other user identifiers seen on the event.
+  synthetic_source_keep: none
   type: keyword
 rule.author:
   dashed_name: rule-author
@@ -13366,6 +13395,7 @@ rule.author:
   normalize:
   - array
   short: Rule author
+  synthetic_source_keep: none
   type: keyword
 rule.category:
   dashed_name: rule-category
@@ -13942,6 +13972,7 @@ server.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 service.address:
   dashed_name: service-address
@@ -14101,6 +14132,7 @@ service.node.roles:
   normalize:
   - array
   short: Roles of the service node.
+  synthetic_source_keep: none
   type: keyword
 service.origin.address:
   dashed_name: service-origin-address
@@ -14257,6 +14289,7 @@ service.origin.node.roles:
   - array
   original_fieldset: service
   short: Roles of the service node.
+  synthetic_source_keep: none
   type: keyword
 service.origin.state:
   dashed_name: service-origin-state
@@ -14466,6 +14499,7 @@ service.target.node.roles:
   - array
   original_fieldset: service
   short: Roles of the service node.
+  synthetic_source_keep: none
   type: keyword
 service.target.state:
   dashed_name: service-target-state
@@ -15009,6 +15043,7 @@ source.user.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 span.id:
   dashed_name: span-id
@@ -15039,6 +15074,7 @@ tags:
   normalize:
   - array
   short: List of keywords used to tag each event.
+  synthetic_source_keep: none
   type: keyword
 threat.enrichments:
   dashed_name: threat-enrichments
@@ -15050,6 +15086,7 @@ threat.enrichments:
   normalize:
   - array
   short: List of objects containing indicators enriching the event.
+  synthetic_source_keep: none
   type: nested
 threat.enrichments.indicator:
   dashed_name: threat-enrichments-indicator
@@ -15158,6 +15195,7 @@ threat.enrichments.indicator.file.attributes:
   - array
   original_fieldset: file
   short: Array of file attributes.
+  synthetic_source_keep: none
   type: keyword
 threat.enrichments.indicator.file.code_signature.digest_algorithm:
   dashed_name: threat-enrichments-indicator-file-code-signature-digest-algorithm
@@ -17755,6 +17793,7 @@ threat.group.alias:
   normalize:
   - array
   short: Alias of the group.
+  synthetic_source_keep: none
   type: keyword
 threat.group.id:
   dashed_name: threat-group-id
@@ -17893,6 +17932,7 @@ threat.indicator.file.attributes:
   - array
   original_fieldset: file
   short: Array of file attributes.
+  synthetic_source_keep: none
   type: keyword
 threat.indicator.file.code_signature.digest_algorithm:
   dashed_name: threat-indicator-file-code-signature-digest-algorithm
@@ -19590,6 +19630,7 @@ threat.indicator.id:
   normalize:
   - array
   short: ID of the indicator
+  synthetic_source_keep: none
   type: keyword
 threat.indicator.ip:
   dashed_name: threat-indicator-ip
@@ -20379,6 +20420,7 @@ threat.software.alias:
   normalize:
   - array
   short: Alias of the software
+  synthetic_source_keep: none
   type: keyword
 threat.software.id:
   dashed_name: threat-software-id
@@ -20430,6 +20472,7 @@ threat.software.platforms:
   normalize:
   - array
   short: Platforms of the software.
+  synthetic_source_keep: none
   type: keyword
 threat.software.reference:
   dashed_name: threat-software-reference
@@ -22036,6 +22079,7 @@ user.changes.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user.domain:
   dashed_name: user-domain
@@ -22179,6 +22223,7 @@ user.effective.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user.email:
   dashed_name: user-email
@@ -22389,6 +22434,7 @@ user.roles:
   - relation: match
     stability: experimental
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user.target.domain:
   dashed_name: user-target-domain
@@ -22520,6 +22566,7 @@ user.target.roles:
   - array
   original_fieldset: user
   short: Array of user roles at the time of the event.
+  synthetic_source_keep: none
   type: keyword
 user_agent.device.name:
   dashed_name: user-agent-device-name
@@ -22887,6 +22934,7 @@ vulnerability.category:
   normalize:
   - array
   short: Category of a vulnerability.
+  synthetic_source_keep: none
   type: keyword
 vulnerability.classification:
   dashed_name: vulnerability-classification