[AWS] Add new pattern to support all 29 vpcflow fields

packages/aws/_dev/build/docs/vpcflow.md

@@ -18,6 +18,12 @@ documentation that can be found in:
 * Custom Format with Traffic Through a Transit Gateway:
   https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html
 
+This integration supports various plain text VPC flow log formats:
+* The default pattern of 14 version 2 fields
+* A custom pattern including all 29 fields, version 2 though 5: `${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${vpc-id} ${subnet-id} ${instance-id} ${tcp-flags} ${type} ${pkt-srcaddr} ${pkt-dstaddr} ${region} ${az-id} ${sublocation-type} ${sublocation-id} ${pkt-src-aws-service} ${pkt-dst-aws-service} ${flow-direction} ${traffic-path}`
+
+**The Parquet format is not supported.**
+
 {{fields "vpcflow"}}
 
 {{event "vpcflow"}}

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.14.3"
+  changes:
+    - description: Add new pattern to VPC Flow logs including all 29 v5 fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2912
 - version: "1.14.2"
   changes:
     - description: Fix billing dashboard.
@@ -13,7 +18,7 @@
   changes:
     - description: Add configuration for max_number_of_messages to the aws.firewall_logs S3 input.
       type: enhancement
-      link: https://github.com/elastic/integrations/pull/
+      link: https://github.com/elastic/integrations/pull/2790
 - version: "1.13.1"
   changes:
     - description: Fix metricbeat- reference in dashboard

packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json

@@ -24,8 +24,8 @@
                     "country_iso_code": "NO",
                     "country_name": "Norway",
                     "location": {
-                        "lat": 62.0,
-                        "lon": 10.0
+                        "lat": 62,
+                        "lon": 10
                     }
                 },
                 "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6",
@@ -65,8 +65,8 @@
                     "country_iso_code": "NO",
                     "country_name": "Norway",
                     "location": {
-                        "lat": 62.0,
-                        "lon": 10.0
+                        "lat": 62,
+                        "lon": 10
                     }
                 },
                 "ip": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6",

packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-v5-all-fields.log

@@ -0,0 +1 @@
+5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 164000066 1640000297 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 - - AMAZON CLOUDFRONT ingress 1

packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-v5-all-fields.log-expected.json

@@ -0,0 +1,101 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2021-12-20T11:38:17.000Z",
+            "aws": {
+                "vpcflow": {
+                    "account_id": "64111117617",
+                    "action": "REJECT",
+                    "instance_id": "i-0axxxxxx1ad77",
+                    "interface_id": "eni-069xxxxxb7a490",
+                    "log_status": "OK",
+                    "pkt_dst_service": "CLOUDFRONT",
+                    "pkt_dstaddr": "10.200.0.80",
+                    "pkt_src_service": "AMAZON",
+                    "pkt_srcaddr": "89.160.20.156",
+                    "sublocation": {},
+                    "subnet_id": "subnet-02d645xxxxxxxdbc0",
+                    "tcp_flags": "1",
+                    "tcp_flags_array": [
+                        "fin"
+                    ],
+                    "traffic_path": "1",
+                    "type": "IPv4",
+                    "version": "5",
+                    "vpc_id": "vpc-09676f97xxxxxb8a7"
+                }
+            },
+            "cloud": {
+                "account": {
+                    "id": "64111117617"
+                },
+                "availability_zone": "use1-az5",
+                "instance": {
+                    "id": "i-0axxxxxx1ad77"
+                },
+                "provider": "aws",
+                "region": "us-east-1"
+            },
+            "destination": {
+                "address": "10.200.0.0",
+                "ip": "10.200.0.0",
+                "port": 33004
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "category": "network_traffic",
+                "end": "2021-12-20T11:38:17.000Z",
+                "kind": "event",
+                "original": "5 64111117617 eni-069xxxxxb7a490 89.160.20.156 10.200.0.0 50041 33004 17 52 1 164000066 1640000297 REJECT OK vpc-09676f97xxxxxb8a7 subnet-02d645xxxxxxxdbc0 i-0axxxxxx1ad77 1 IPv4 89.160.20.156 10.200.0.80 us-east-1 use1-az5 - - AMAZON CLOUDFRONT ingress 1",
+                "outcome": "deny",
+                "start": "1975-03-14T03:34:26.000Z",
+                "type": "flow"
+            },
+            "network": {
+                "bytes": 1,
+                "community_id": "1:+zLdyx7gcy2L5ERP4Je5AZMgl3A=",
+                "direction": "ingress",
+                "iana_number": "17",
+                "packets": 52,
+                "transport": "udp",
+                "type": "ipv4"
+            },
+            "related": {
+                "ip": [
+                    "89.160.20.156",
+                    "10.200.0.0"
+                ]
+            },
+            "source": {
+                "address": "89.160.20.156",
+                "as": {
+                    "number": 29518,
+                    "organization": {
+                        "name": "Bredband2 AB"
+                    }
+                },
+                "bytes": 1,
+                "geo": {
+                    "city_name": "Linköping",
+                    "continent_name": "Europe",
+                    "country_iso_code": "SE",
+                    "country_name": "Sweden",
+                    "location": {
+                        "lat": 58.4167,
+                        "lon": 15.6167
+                    },
+                    "region_iso_code": "SE-E",
+                    "region_name": "Östergötland County"
+                },
+                "ip": "89.160.20.156",
+                "packets": 52,
+                "port": 50041
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}

packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml

@@ -39,6 +39,10 @@ processors:
       field: event.original
       pattern: '%{aws.vpcflow.version} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.account_id} %{aws.vpcflow.type} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{aws.vpcflow.protocol} %{aws.vpcflow.bytes} %{aws.vpcflow.packets} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.log_status}'
       if: ctx?._temp_?.message_token_count == 21
+  - dissect:
+      field: event.original
+      pattern: '%{aws.vpcflow.version} %{aws.vpcflow.account_id} %{aws.vpcflow.interface_id} %{aws.vpcflow.srcaddr} %{aws.vpcflow.dstaddr} %{aws.vpcflow.srcport} %{aws.vpcflow.dstport} %{aws.vpcflow.protocol} %{aws.vpcflow.packets} %{aws.vpcflow.bytes} %{aws.vpcflow.start} %{aws.vpcflow.end} %{aws.vpcflow.action} %{aws.vpcflow.log_status} %{aws.vpcflow.vpc_id} %{aws.vpcflow.subnet_id} %{aws.vpcflow.instance_id} %{aws.vpcflow.tcp_flags} %{aws.vpcflow.type} %{aws.vpcflow.pkt_srcaddr} %{aws.vpcflow.pkt_dstaddr} %{cloud.region} %{cloud.availability_zone} %{aws.vpcflow.sublocation.type} %{aws.vpcflow.sublocation.id} %{aws.vpcflow.pkt_src_service} %{aws.vpcflow.pkt_dst_service} %{network.direction} %{aws.vpcflow.traffic_path}'
+      if: ctx?._temp_?.message_token_count == 29
 
   # Convert Unix epoch to timestamp
   - date:

packages/aws/data_stream/vpcflow/fields/ecs.yml

@@ -58,6 +58,8 @@
   external: ecs
 - name: network.type
   external: ecs
+- name: network.direction
+  external: ecs
 - name: related.ip
   external: ecs
 - name: source.address

packages/aws/data_stream/vpcflow/fields/fields.yml

@@ -54,3 +54,23 @@
       type: keyword
       description: |
         The type of traffic: IPv4, IPv6, or EFA.
+    - name: pkt_dst_service
+      type: keyword
+      description: |
+        The name of the subset of IP address ranges for the pkt-dstaddr field, if the source IP address is for an AWS service.
+    - name: pkt_src_service
+      type: keyword
+      description: |
+        The name of the subset of IP address ranges for the pkt-srcaddr field, if the source IP address is for an AWS service.
+    - name: traffic_path
+      type: keyword
+      description: |
+        The path that egress traffic takes to the destination. To determine whether the traffic is egress traffic, check the `network.direction` field. The possible values can be found [here](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html#flow-logs-fields). If none of the values apply, the field is set to -.
+    - name: sublocation.type
+      type: keyword
+      description: |
+        The type of sublocation that's returned in the sublocation-id field. The possible values are: wavelength | outpost | localzone. If the traffic is not from a sublocation, the field is removed.
+    - name: sublocation.id
+      type: keyword
+      description: |
+        The ID of the sublocation that contains the network interface for which traffic is recorded. If the traffic is not from a sublocation, the field is removed.

packages/aws/docs/vpcflow.md

@@ -18,6 +18,12 @@ documentation that can be found in:
 * Custom Format with Traffic Through a Transit Gateway:
   https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-records-examples.html
 
+This integration supports various plain text VPC flow log formats:
+* The default pattern of 14 version 2 fields
+* A custom pattern including all 29 fields, version 2 though 5: `${version} ${account-id} ${interface-id} ${srcaddr} ${dstaddr} ${srcport} ${dstport} ${protocol} ${packets} ${bytes} ${start} ${end} ${action} ${log-status} ${vpc-id} ${subnet-id} ${instance-id} ${tcp-flags} ${type} ${pkt-srcaddr} ${pkt-dstaddr} ${region} ${az-id} ${sublocation-type} ${sublocation-id} ${pkt-src-aws-service} ${pkt-dst-aws-service} ${flow-direction} ${traffic-path}`
+
+**The Parquet format is not supported.**
+
 **Exported fields**
 
 | Field | Description | Type |
@@ -28,11 +34,16 @@ documentation that can be found in:
 | aws.vpcflow.instance_id | The ID of the instance that's associated with network interface for which the traffic is recorded, if the instance is owned by you. | keyword |
 | aws.vpcflow.interface_id | The ID of the network interface for which the traffic is recorded. | keyword |
 | aws.vpcflow.log_status | The logging status of the flow log, OK, NODATA or SKIPDATA. | keyword |
+| aws.vpcflow.pkt_dst_service | The name of the subset of IP address ranges for the pkt-dstaddr field, if the source IP address is for an AWS service. | keyword |
 | aws.vpcflow.pkt_dstaddr | The packet-level (original) destination IP address for the traffic. | ip |
+| aws.vpcflow.pkt_src_service | The name of the subset of IP address ranges for the pkt-srcaddr field, if the source IP address is for an AWS service. | keyword |
 | aws.vpcflow.pkt_srcaddr | The packet-level (original) source IP address of the traffic. | ip |
+| aws.vpcflow.sublocation.id | The ID of the sublocation that contains the network interface for which traffic is recorded. If the traffic is not from a sublocation, the field is removed. | keyword |
+| aws.vpcflow.sublocation.type | The type of sublocation that's returned in the sublocation-id field. The possible values are: wavelength | outpost | localzone. If the traffic is not from a sublocation, the field is removed. | keyword |
 | aws.vpcflow.subnet_id | The ID of the subnet that contains the network interface for which the traffic is recorded. | keyword |
 | aws.vpcflow.tcp_flags | The bitmask value for the following TCP flags: 2=SYN,18=SYN-ACK,1=FIN,4=RST | keyword |
 | aws.vpcflow.tcp_flags_array | List of TCP flags: 'fin, syn, rst, psh, ack, urg' | keyword |
+| aws.vpcflow.traffic_path | The path that egress traffic takes to the destination. To determine whether the traffic is egress traffic, check the `network.direction` field. The possible values can be found [here](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs.html#flow-logs-fields). If none of the values apply, the field is set to -. | keyword |
 | aws.vpcflow.type | The type of traffic: IPv4, IPv6, or EFA. | keyword |
 | aws.vpcflow.version | The VPC Flow Logs version. If you use the default format, the version is 2. If you specify a custom format, the version is 3. | keyword |
 | aws.vpcflow.vpc_id | The ID of the VPC that contains the network interface for which the traffic is recorded. | keyword |
@@ -95,6 +106,7 @@ documentation that can be found in:
 | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword |
 | network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long |
 | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword |
+| network.direction | Direction of the network traffic. Recommended values are:   \* ingress   \* egress   \* inbound   \* outbound   \* internal   \* external   \* unknown  When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword |
 | network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword |
 | network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long |
 | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword |

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: aws
 title: AWS
-version: 1.14.2
+version: 1.14.3
 license: basic
 description: Collect logs and metrics from Amazon Web Services with Elastic Agent.
 type: integration