Optimize latest transform source queries across CDR packages

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "6.5.0"
+  changes:
+    - description: Add timestamp range filter and storage tier exclusion to latest transform source queries to reduce scan scope and improve performance.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18098
 - version: "6.4.3"
   changes:
     - description: Align `event.kind` with ECS in `securityhub_findings_full_posture` data stream to avoid conflicts with ingest pipeline values.

packages/aws/data_stream/inspector/_dev/deploy/docker/files/config.yml

@@ -15,7 +15,7 @@ rules:
                 },
                 "exploitAvailable": "NO",
                 "findingArn": "arn:aws:inspector2:us-east-2:123456789012:finding/fb6294abcdef0123456789abcdef8123",
-                "firstObservedAt": 1748539687.919,
+                "firstObservedAt": 1773422887.919,
                 "fixAvailable": "YES",
                 "inspectorScore": 6.5,
                 "inspectorScoreDetails": {
@@ -28,7 +28,7 @@ rules:
                     "version": "3.1"
                   }
                 },
-                "lastObservedAt": 1749165796.162,
+                "lastObservedAt": 1774048996.162,
                 "packageVulnerabilityDetails": {
                   "cvss": [
                     {
@@ -59,9 +59,9 @@ rules:
                   "relatedVulnerabilities": [],
                   "source": "NVD",
                   "sourceUrl": "https://nvd.nist.gov/vuln/detail/CVE-2025-22872",
-                  "vendorCreatedAt": 1744827364,
+                  "vendorCreatedAt": 1769710564,
                   "vendorSeverity": "MEDIUM",
-                  "vendorUpdatedAt": 1747437319,
+                  "vendorUpdatedAt": 1772320519,
                   "vulnerabilityId": "CVE-2025-22872",
                   "vulnerablePackages": [
                     {
@@ -149,7 +149,7 @@ rules:
                           "10.90.1.206"
                         ],
                         "ipV6Addresses": [],
-                        "launchedAt": 1748534768,
+                        "launchedAt": 1773417968,
                         "platform": "AMAZON_LINUX_2",
                         "subnetId": "subnet-0ababcdefabcdef8b",
                         "type": "t3.medium",
@@ -176,7 +176,7 @@ rules:
                 "status": "ACTIVE",
                 "title": "CVE-2025-22872 - golang.org/x/net, golang.org/x/net and 4 more",
                 "type": "PACKAGE_VULNERABILITY",
-                "updatedAt": 1749165796.162
+                "updatedAt": 1774048996.162
               }
             ]
           }

packages/aws/elasticsearch/transform/latest_cdr_misconfigurations/transform.yml

@@ -3,9 +3,17 @@ source:
     - "logs-aws.securityhub_findings_full_posture-*"
   query:
     bool:
+      must:
+        - range:
+            "@timestamp":
+              gte: "now-26h"
       must_not:
-        exists:
-          field: error.message
+        - exists:
+            field: error.message
+        - terms:
+            _tier:
+              - data_frozen
+              - data_cold
 dest:
   index: "security_solution-aws.misconfiguration_latest-v3"
   aliases:
@@ -25,11 +33,11 @@ sync:
 retention_policy:
   time:
     field: "@timestamp"
-    max_age: 24h
+    max_age: 26h
 settings:
   unattended: true
 _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.3.0
+  fleet_transform_version: 0.4.0

packages/aws/elasticsearch/transform/latest_cdr_misconfigurations_awsconfig/transform.yml

@@ -4,9 +4,17 @@ source:
     - "logs-aws.config-*"
   query:
     bool:
+      must:
+        - range:
+            "@timestamp":
+              gte: "now-26h"
       must_not:
-        exists:
-          field: error.message
+        - exists:
+            field: error.message
+        - terms:
+            _tier:
+              - data_frozen
+              - data_cold
 dest:
   index: "security_solution-awsconfig.misconfiguration_latest-v1"
   aliases:
@@ -30,9 +38,9 @@ sync:
 retention_policy:
   time:
     field: "@timestamp"
-    max_age: 24h
+    max_age: 26h
 _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during
   # package installation.
-  fleet_transform_version: 0.1.1
+  fleet_transform_version: 0.2.0

packages/aws/elasticsearch/transform/latest_cdr_vulnerabilities_awsinspector/transform.yml

@@ -8,9 +8,16 @@ source:
             aws.inspector.status: ACTIVE
         - match:
             aws.inspector.type: PACKAGE_VULNERABILITY
+        - range:
+            "@timestamp":
+              gte: "now-90d"
       must_not:
-        exists:
-          field: error.message
+        - exists:
+            field: error.message
+        - terms:
+            _tier:
+              - data_frozen
+              - data_cold
 dest:
   index: "security_solution-awsinspector.vulnerability_latest-v1"
   aliases:
@@ -36,4 +43,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.1
+  fleet_transform_version: 0.2.0

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.4.0
 name: aws
 title: AWS
-version: 6.4.3
+version: 6.5.0
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:

packages/aws_securityhub/_dev/deploy/docker/files/config.yml

@@ -48,15 +48,15 @@ rules:
                   "region": "us-east-2"
                 },
                 "finding_info": {
-                  "created_time": 1758662368239,
-                  "created_time_dt": "2025-09-23T21:19:28.239Z",
+                  "created_time": 1774214368239,
+                  "created_time_dt": "2026-03-22T21:19:28.239Z",
                   "desc": "If the PATH environment variable contains paths which are executables (rather than just directories), passing certain strings to LookPath (\"\", \".\", and \"..\"), can result in the binaries listed in the PATH being unexpectedly returned.",
-                  "first_seen_time": 1758662368239,
-                  "first_seen_time_dt": "2025-09-23T21:19:28.239Z",
-                  "last_seen_time": 1758662368239,
-                  "last_seen_time_dt": "2025-09-23T21:19:28.239Z",
-                  "modified_time": 1758662368239,
-                  "modified_time_dt": "2025-09-23T21:19:28.239Z",
+                  "first_seen_time": 1774214368239,
+                  "first_seen_time_dt": "2026-03-22T21:19:28.239Z",
+                  "last_seen_time": 1774214368239,
+                  "last_seen_time_dt": "2026-03-22T21:19:28.239Z",
+                  "modified_time": 1774214368239,
+                  "modified_time_dt": "2026-03-22T21:19:28.239Z",
                   "title": "CVE-2025-47906 - go/stdlib",
                   "types": [
                     "Software and Configuration Checks/Vulnerabilities/CVE"
@@ -87,7 +87,7 @@ rules:
                         "codeSha256": "OBDFNBKKvsf/dfbSJFBVDKB64v1sJUYVJH/SV5nsdv5svssv=",
                         "executionRoleArn": "arn:aws:iam::123456789012:role/cloudtrail-test-ab-LambdaExecutionRole-jVHJUFvbhIHVD6",
                         "functionName": "cloudtrail-test-ab-LambdaFunction-jVHJUsdbdkj6",
-                        "lastModifiedAt": "2025-08-12T08:23:50Z",
+                        "lastModifiedAt": "2026-02-08T08:23:50Z",
                         "layers": [
                           "arn:aws:lambda:us-east-2:267093732750:layer:test-cloud-forwarder-ver-0-2-0:1"
                         ],
@@ -126,8 +126,8 @@ rules:
                 "severity_id": 3,
                 "status": "New",
                 "status_id": 1,
-                "time": 1758662368239,
-                "time_dt": "2025-09-23T21:19:28.239Z",
+                "time": 1774214368239,
+                "time_dt": "2026-03-22T21:19:28.239Z",
                 "type_name": "Vulnerability Finding: Create",
                 "type_uid": 200201,
                 "vendor_attributes": {
@@ -147,8 +147,8 @@ rules:
                       }
                     ],
                     "cve": {
-                      "created_time": 1758222937000,
-                      "created_time_dt": "2025-09-18T19:15:37.000Z",
+                      "created_time": 1773774937000,
+                      "created_time_dt": "2026-03-17T19:15:37.000Z",
                       "cvss": [
                         {
                           "base_score": 6.5,
@@ -162,8 +162,8 @@ rules:
                       "epss": {
                         "score": "0.00017"
                       },
-                      "modified_time": 1758297627000,
-                      "modified_time_dt": "2025-09-19T16:00:27.000Z",
+                      "modified_time": 1773849627000,
+                      "modified_time_dt": "2026-03-18T16:00:27.000Z",
                       "uid": "CVE-2025-47906"
                     },
                     "fix_coverage": "Complete",
@@ -205,15 +205,15 @@ rules:
                     "type": "Rule",
                     "type_id": 1
                   },
-                  "created_time": 1758273553640,
-                  "created_time_dt": "2025-09-19T09:19:13.640Z",
+                  "created_time": 1773825553640,
+                  "created_time_dt": "2026-03-18T09:19:13.640Z",
                   "desc": "Checks whether the default version of IAM customer managed policies allow principals to use the AWS Key Management Service (KMS) decryption actions on all resources. This control fails if kms:Decrypt or kms:ReEncryptFrom actions are allowed on all KMS keys. The control evaluates both attached and unattached customer managed policies. It does not check inline policies or AWS managed policies.",
-                  "first_seen_time": 1758273549591,
-                  "first_seen_time_dt": "2025-09-19T09:19:09.591Z",
-                  "last_seen_time": 1758273549591,
-                  "last_seen_time_dt": "2025-09-19T09:19:09.591Z",
-                  "modified_time": 1758273553640,
-                  "modified_time_dt": "2025-09-19T09:19:13.640Z",
+                  "first_seen_time": 1773825549591,
+                  "first_seen_time_dt": "2026-03-18T09:19:09.591Z",
+                  "last_seen_time": 1773825549591,
+                  "last_seen_time_dt": "2026-03-18T09:19:09.591Z",
+                  "modified_time": 1773825553640,
+                  "modified_time_dt": "2026-03-18T09:19:13.640Z",
                   "title": "IAM customer managed policies should not allow decryption actions on all KMS keys",
                   "types": [
                     "Software and Configuration Checks/Industry and Regulatory Standards",
@@ -251,8 +251,8 @@ rules:
                 "severity_id": 1,
                 "status": "Resolved",
                 "status_id": 4,
-                "time": 1758273553640,
-                "time_dt": "2025-09-19T09:19:13.640Z",
+                "time": 1773825553640,
+                "time_dt": "2026-03-18T09:19:13.640Z",
                 "type_name": "Compliance Finding: Create",
                 "type_uid": 200301,
                 "vendor_attributes": { "severity": "Informational", "severity_id": 1 }
@@ -300,15 +300,15 @@ rules:
                     "type": "Rule",
                     "type_id": 1
                   },
-                  "created_time": 1758273439594,
-                  "created_time_dt": "2025-09-19T09:17:19.594Z",
+                  "created_time": 1773825439594,
+                  "created_time_dt": "2026-03-18T09:17:19.594Z",
                   "desc": "This controls checks whether an Amazon SQS access policy allows public access to an SQS queue. The control fails if an SQS access policy allows public access to the queue.",
-                  "first_seen_time": 1758273437503,
-                  "first_seen_time_dt": "2025-09-19T09:17:17.503Z",
-                  "last_seen_time": 1758273437503,
-                  "last_seen_time_dt": "2025-09-19T09:17:17.503Z",
-                  "modified_time": 1758273439594,
-                  "modified_time_dt": "2025-09-19T09:17:19.594Z",
+                  "first_seen_time": 1773825437503,
+                  "first_seen_time_dt": "2026-03-18T09:17:17.503Z",
+                  "last_seen_time": 1773825437503,
+                  "last_seen_time_dt": "2026-03-18T09:17:17.503Z",
+                  "modified_time": 1773825439594,
+                  "modified_time_dt": "2026-03-18T09:17:19.594Z",
                   "title": "SQS queue access policies should not allow public access",
                   "types": [
                     "Software and Configuration Checks/Industry and Regulatory Standards",
@@ -346,8 +346,8 @@ rules:
                 "severity_id": 1,
                 "status": "Resolved",
                 "status_id": 4,
-                "time": 1758273439594,
-                "time_dt": "2025-09-19T09:17:19.594Z",
+                "time": 1773825439594,
+                "time_dt": "2026-03-18T09:17:19.594Z",
                 "type_name": "Compliance Finding: Create",
                 "type_uid": 200301,
                 "vendor_attributes": { "severity": "Informational", "severity_id": 1 }
@@ -379,15 +379,15 @@ rules:
                     "type": "Rule",
                     "type_id": 1
                   },
-                  "created_time": 1758273447903,
-                  "created_time_dt": "2025-09-19T09:17:27.903Z",
+                  "created_time": 1773825447903,
+                  "created_time_dt": "2026-03-18T09:17:27.903Z",
                   "desc": "This control checks whether Amazon SQS queues are encrypted at rest.",
-                  "first_seen_time": 1758273437473,
-                  "first_seen_time_dt": "2025-09-19T09:17:17.473Z",
-                  "last_seen_time": 1758273437473,
-                  "last_seen_time_dt": "2025-09-19T09:17:17.473Z",
-                  "modified_time": 1758273447903,
-                  "modified_time_dt": "2025-09-19T09:17:27.903Z",
+                  "first_seen_time": 1773825437473,
+                  "first_seen_time_dt": "2026-03-18T09:17:17.473Z",
+                  "last_seen_time": 1773825437473,
+                  "last_seen_time_dt": "2026-03-18T09:17:17.473Z",
+                  "modified_time": 1773825447903,
+                  "modified_time_dt": "2026-03-18T09:17:27.903Z",
                   "title": "Amazon SQS queues should be encrypted at rest",
                   "types": [
                     "Software and Configuration Checks/Industry and Regulatory Standards",
@@ -425,8 +425,8 @@ rules:
                 "severity_id": 3,
                 "status": "New",
                 "status_id": 1,
-                "time": 1758273447903,
-                "time_dt": "2025-09-19T09:17:27.903Z",
+                "time": 1773825447903,
+                "time_dt": "2026-03-18T09:17:27.903Z",
                 "type_name": "Compliance Finding: Create",
                 "type_uid": 200301,
                 "vendor_attributes": { "severity": "Medium", "severity_id": 3 }

packages/aws_securityhub/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.3.0"
+  changes:
+    - description: Add timestamp range filter and storage tier exclusion to latest transform source queries to reduce scan scope and improve performance.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/18098
 - version: "0.2.1"
   changes:
     - description: Use triple-brace Mustache templating when referencing variables in ingest pipelines.

packages/aws_securityhub/elasticsearch/transform/latest_cdr_vulnerabilities/transform.yml

@@ -8,11 +8,18 @@ source:
             aws_securityhub.finding.class_uid: '2002' # Vulnerability finding
         - exists:
             field: package.name
+        - range:
+            "@timestamp":
+              gte: "now-90d"
       must_not:
         - match:
             aws_securityhub.finding.activity_id: '3' # Close
         - exists:
             field: error.message
+        - terms:
+            _tier:
+              - data_frozen
+              - data_cold
 dest:
   index: "security_solution-aws_securityhub.vulnerability_latest-v1"
   aliases:
@@ -38,4 +45,4 @@ _meta:
   managed: true
   # Bump this version to delete, reinstall, and restart the transform during package.
   # Version bump is needed if there is any code change in transform.
-  fleet_transform_version: 0.1.0
+  fleet_transform_version: 0.2.0

packages/aws_securityhub/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.5.0
 name: aws_securityhub
 title: "AWS Security Hub"
-version: 0.2.1
+version: 0.3.0
 source:
   license: "Elastic-2.0"
 description: Collect logs from AWS Security Hub with Elastic Agent.