From ae2d66958504e091cfa00c4c66dc55a447bb7bf1 Mon Sep 17 00:00:00 2001 From: jdv Date: Mon, 7 Sep 2026 16:38:35 +0200 Subject: [PATCH] datasources schema - AI generated --- pkg/acquisition/schemas/appsec.yaml | 140 ++++++++++++++++++ pkg/acquisition/schemas/cloudwatch.yaml | 126 ++++++++++++++++ pkg/acquisition/schemas/file.yaml | 114 +++++++++++++++ pkg/acquisition/schemas/http.yaml | 171 ++++++++++++++++++++++ pkg/acquisition/schemas/journalctl.yaml | 72 +++++++++ pkg/acquisition/schemas/k8s-audit.yaml | 85 +++++++++++ pkg/acquisition/schemas/kafka.yaml | 130 ++++++++++++++++ pkg/acquisition/schemas/kinesis.yaml | 113 ++++++++++++++ pkg/acquisition/schemas/loki.yaml | 121 +++++++++++++++ pkg/acquisition/schemas/s3.yaml | 129 ++++++++++++++++ pkg/acquisition/schemas/syslog.yaml | 88 +++++++++++ pkg/acquisition/schemas/victorialogs.yaml | 113 ++++++++++++++ pkg/acquisition/schemas/wineventlog.yaml | 97 ++++++++++++ 13 files changed, 1499 insertions(+) create mode 100644 pkg/acquisition/schemas/appsec.yaml create mode 100644 pkg/acquisition/schemas/cloudwatch.yaml create mode 100644 pkg/acquisition/schemas/file.yaml create mode 100644 pkg/acquisition/schemas/http.yaml create mode 100644 pkg/acquisition/schemas/journalctl.yaml create mode 100644 pkg/acquisition/schemas/k8s-audit.yaml create mode 100644 pkg/acquisition/schemas/kafka.yaml create mode 100644 pkg/acquisition/schemas/kinesis.yaml create mode 100644 pkg/acquisition/schemas/loki.yaml create mode 100644 pkg/acquisition/schemas/s3.yaml create mode 100644 pkg/acquisition/schemas/syslog.yaml create mode 100644 pkg/acquisition/schemas/victorialogs.yaml create mode 100644 pkg/acquisition/schemas/wineventlog.yaml diff --git a/pkg/acquisition/schemas/appsec.yaml b/pkg/acquisition/schemas/appsec.yaml new file mode 100644 index 00000000000..81327187f78 --- /dev/null +++ b/pkg/acquisition/schemas/appsec.yaml @@ -0,0 +1,140 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec AppSec datasource +description: > + Schema for appsec acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/appsec.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: appsec + description: > + Must be appsec to bind this acquisition entry to the AppSec WAF datasource. + mode: + type: string + enum: [tail] + default: tail + description: > + Acquisition mode (only tail streaming is supported). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: appsec). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: > + Friendly identifier for the datasource entry; defaults to listen_addr + and path (or listen_socket) when omitted. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + listen_addr: + type: string + default: "127.0.0.1:7422" + description: > + Address:port to bind the AppSec HTTP listener on. Mutually exclusive + with listen_socket. + listen_socket: + type: string + description: > + Unix socket path to bind the AppSec listener on, instead of listen_addr. + cert_file: + type: string + description: TLS certificate file for the AppSec listener. + key_file: + type: string + description: TLS key file for the AppSec listener. + path: + type: string + default: "/" + description: > + HTTP path the bouncer posts requests to; a leading / is added if missing. + routines: + type: integer + minimum: 1 + default: 1 + description: > + Number of parallel appsec runner goroutines. + appsec_config: + type: string + description: > + Single appsec-config item (or glob pattern) to load. Mutually exclusive + with appsec_configs and appsec_config_path. + appsec_configs: + type: array + minItems: 1 + items: + type: string + minLength: 1 + description: > + List of appsec-config items (or glob patterns) to load. Mutually + exclusive with appsec_config and appsec_config_path. + appsec_config_path: + type: string + description: > + Path to an appsec-config file to load directly. Mutually exclusive + with appsec_config and appsec_configs. + auth_cache_duration: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 1m + description: > + How long a validated bouncer API key is cached before being re-checked + against LAPI. + auth_timeout: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 200ms + description: > + Timeout for the LAPI round-trip that validates a bouncer API key. 0 + disables the timeout. + body_read_timeout: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 1s + description: > + Timeout for reading the bouncer request body. 0 disables the timeout. +required: + - source +allOf: + - description: > + Exactly one appsec-config source must be provided, matching the + UnmarshalConfig checks. + not: + anyOf: + - required: [appsec_config, appsec_configs] + - required: [appsec_config_path, appsec_configs] + - anyOf: + - required: [appsec_config] + - required: [appsec_configs] + - required: [appsec_config_path] +examples: + - source: appsec + listen_addr: 127.0.0.1:7422 + appsec_config: crowdsecurity/vpatch-generic-rules + labels: + type: appsec diff --git a/pkg/acquisition/schemas/cloudwatch.yaml b/pkg/acquisition/schemas/cloudwatch.yaml new file mode 100644 index 00000000000..dec1028500a --- /dev/null +++ b/pkg/acquisition/schemas/cloudwatch.yaml @@ -0,0 +1,126 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Cloudwatch datasource +description: > + Schema for cloudwatch acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/cloudwatch.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: cloudwatch + description: > + Must be cloudwatch to bind this acquisition entry to the Cloudwatch + datasource. + mode: + type: string + enum: [tail, cat] + default: tail + description: > + Acquisition mode (tail streams logs, cat performs a finite read). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: cloudwatch). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + group_name: + type: string + minLength: 1 + description: The Cloudwatch log group to monitor. + stream_regexp: + type: string + format: regex + description: > + Regular expression used to select streams within the log group. + Mutually exclusive in practice with stream_name. + stream_name: + type: string + description: > + Exact stream name to follow within the log group. + describelogstreams_limit: + type: integer + minimum: 1 + description: > + Batch size for the DescribeLogStreams pagination. + getlogeventspages_limit: + type: integer + minimum: 1 + description: > + Batch size for the GetLogEvents pagination. + poll_new_stream_interval: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: > + Frequency at which new streams are discovered within the log group. + max_stream_age: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: > + Only monitor streams that have been updated within this duration. + poll_stream_interval: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: Frequency at which each stream is polled. + stream_read_timeout: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: > + Stop monitoring a stream that hasn't been updated within this + duration; it may be reopened later. + aws_api_timeout: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: Timeout applied to AWS API calls. + aws_profile: + type: string + description: Named AWS profile to use for credentials. + prepend_cloudwatch_timestamp: + type: boolean + description: > + Prepend the Cloudwatch event timestamp to the log line. + aws_config_dir: + type: string + description: > + Directory containing an AWS config/credentials pair, used instead of + aws_region and the default credential chain. + aws_region: + type: string + description: > + AWS region to use; required unless aws_config_dir is set. +required: + - source + - group_name +examples: + - source: cloudwatch + group_name: /my/log/group + aws_region: eu-west-1 + labels: + type: cloudwatch diff --git a/pkg/acquisition/schemas/file.yaml b/pkg/acquisition/schemas/file.yaml new file mode 100644 index 00000000000..5a173283e89 --- /dev/null +++ b/pkg/acquisition/schemas/file.yaml @@ -0,0 +1,114 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec File datasource +description: > + Schema for file acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/file.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: file + description: > + Must be file to bind this acquisition entry to the File datasource. + mode: + type: string + enum: [tail, cat] + default: tail + description: > + Acquisition mode (tail streams new lines, cat reads the file(s) once). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: syslog). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + filenames: + type: array + minItems: 1 + items: + type: string + minLength: 1 + description: > + Glob patterns of files to read/tail. At least one of filenames or + filename is required. + filename: + type: string + minLength: 1 + description: > + Single glob pattern, appended to filenames when set. + exclude_regexps: + type: array + minItems: 1 + items: + type: string + minLength: 1 + format: regex + description: > + Regular expressions used to exclude matched files from acquisition. + force_inotify: + type: boolean + default: false + description: > + Force an inotify watch on the parent directory even for non-glob + patterns. + max_buffer_size: + type: integer + minimum: 1 + description: > + Maximum size, in bytes, of a scanned line; defaults to bufio's + MaxScanTokenSize when unset. + poll_without_inotify: + type: boolean + description: > + Fall back to polling instead of inotify (auto-detected on some + filesystems such as network shares when unset). + discovery_poll_enable: + type: boolean + default: false + description: > + Periodically re-glob patterns to discover newly created files matching + them, in addition to inotify-based discovery. + discovery_poll_interval: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: > + Interval between discovery polls when discovery_poll_enable is true. +required: + - source +anyOf: + - required: [filenames] + - required: [filename] +examples: + - source: file + filenames: + - /var/log/nginx/access.log + labels: + type: nginx diff --git a/pkg/acquisition/schemas/http.yaml b/pkg/acquisition/schemas/http.yaml new file mode 100644 index 00000000000..bbbfe60b240 --- /dev/null +++ b/pkg/acquisition/schemas/http.yaml @@ -0,0 +1,171 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec HTTP datasource +description: > + Schema for http acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/http.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: http + description: > + Must be http to bind this acquisition entry to the HTTP datasource. + mode: + type: string + enum: [tail] + default: tail + description: > + Acquisition mode (only tail streaming is supported). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: http). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + listen_addr: + type: string + description: > + Address:port to bind the HTTP listener on. Required unless + listen_socket is set. + listen_socket: + type: string + description: > + Unix socket path to bind the HTTP listener on, instead of listen_addr. + path: + type: string + default: "/" + minLength: 1 + description: HTTP path this datasource listens on; must start with /. + auth_type: + type: string + enum: [basic_auth, headers, mtls] + description: > + Authentication method required from callers. + basic_auth: + type: object + additionalProperties: false + description: Required when auth_type is basic_auth. + properties: + username: + type: string + minLength: 1 + password: + type: string + minLength: 1 + required: + - username + - password + headers: + type: object + minProperties: 1 + additionalProperties: + type: string + description: > + Expected header/value pairs; required when auth_type is headers. + tls: + type: object + additionalProperties: false + description: > + TLS listener settings; ca_cert is required when auth_type is mtls. + properties: + insecure_skip_verify: + type: boolean + default: false + server_cert: + type: string + minLength: 1 + server_key: + type: string + minLength: 1 + ca_cert: + type: string + required: + - server_cert + - server_key + custom_status_code: + type: integer + description: > + HTTP status code returned on successful ingestion, in place of the + default. + custom_headers: + type: object + additionalProperties: + type: string + description: Extra response headers to send back to the caller. + max_body_size: + type: integer + minimum: 1 + default: 10485760 + description: > + Maximum accepted request body size, in bytes, after decompression. + timeout: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: Request handling timeout. +required: + - source + - auth_type +anyOf: + - required: [listen_addr] + - required: [listen_socket] +allOf: + - if: + properties: + auth_type: + const: basic_auth + required: [auth_type] + then: + required: [basic_auth] + - if: + properties: + auth_type: + const: headers + required: [auth_type] + then: + required: [headers] + - if: + properties: + auth_type: + const: mtls + required: [auth_type] + then: + required: [tls] + properties: + tls: + required: [ca_cert] +examples: + - source: http + listen_addr: 127.0.0.1:8080 + auth_type: headers + headers: + X-Api-Key: secret + labels: + type: http diff --git a/pkg/acquisition/schemas/journalctl.yaml b/pkg/acquisition/schemas/journalctl.yaml new file mode 100644 index 00000000000..b4352281d22 --- /dev/null +++ b/pkg/acquisition/schemas/journalctl.yaml @@ -0,0 +1,72 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Journalctl datasource +description: > + Schema for journalctl acquisition entries consumed by CrowdSec. Every + field mirrors pkg/acquisition/modules/journalctl.Configuration and the + embedded configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: journalctl + description: > + Must be journalctl to bind this acquisition entry to the Journalctl + datasource. + mode: + type: string + enum: [tail, cat] + default: tail + description: > + Acquisition mode (tail streams new entries, cat performs a finite read). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: syslog). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + journalctl_filter: + type: array + minItems: 1 + items: + type: string + minLength: 1 + description: > + Arguments appended to the journalctl command line (for example + ["-u", "ssh.service"]). +required: + - source + - journalctl_filter +examples: + - source: journalctl + journalctl_filter: + - "-u" + - ssh.service + labels: + type: syslog diff --git a/pkg/acquisition/schemas/k8s-audit.yaml b/pkg/acquisition/schemas/k8s-audit.yaml new file mode 100644 index 00000000000..05bd82e1585 --- /dev/null +++ b/pkg/acquisition/schemas/k8s-audit.yaml @@ -0,0 +1,85 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Kubernetes audit datasource +description: > + Schema for k8s-audit acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/kubernetesaudit.Configuration and the + embedded configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: k8s-audit + description: > + Must be k8s-audit to bind this acquisition entry to the Kubernetes + audit webhook datasource. + mode: + type: string + enum: [tail] + default: tail + description: > + Acquisition mode (only tail streaming is supported). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: k8s-audit). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + listen_addr: + type: string + minLength: 1 + description: Address the audit webhook HTTP server binds to. + listen_port: + type: integer + minimum: 1 + maximum: 65535 + description: Port the audit webhook HTTP server binds to. + webhook_path: + type: string + minLength: 1 + description: > + HTTP path the Kubernetes audit webhook posts to; a leading / is added + if missing. + max_body_size: + type: integer + minimum: 1 + default: 10485760 + description: Maximum accepted request body size, in bytes. +required: + - source + - listen_addr + - listen_port + - webhook_path +examples: + - source: k8s-audit + listen_addr: 0.0.0.0 + listen_port: 8080 + webhook_path: /webhook + labels: + type: k8s-audit diff --git a/pkg/acquisition/schemas/kafka.yaml b/pkg/acquisition/schemas/kafka.yaml new file mode 100644 index 00000000000..e1497d322aa --- /dev/null +++ b/pkg/acquisition/schemas/kafka.yaml @@ -0,0 +1,130 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Kafka datasource +description: > + Schema for kafka acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/kafka.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: kafka + description: > + Must be kafka to bind this acquisition entry to the Kafka datasource. + mode: + type: string + enum: [tail] + default: tail + description: > + Acquisition mode (only tail streaming is supported). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: kafka). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + brokers: + type: array + minItems: 1 + items: + type: string + minLength: 1 + description: List of Kafka broker addresses to connect to. + topic: + type: string + minLength: 1 + description: Kafka topic to consume from. + group_id: + type: string + description: > + Consumer group ID. Mutually exclusive with partition. + partition: + type: integer + description: > + Explicit non-zero partition to read from. Mutually exclusive with + group_id (only one may be set to a non-zero/non-empty value). + timeout: + type: string + pattern: "^[0-9]+$" + description: > + Dial timeout in seconds (defaults to 10 when unset). + tls: + type: object + additionalProperties: false + description: TLS settings for the Kafka connection. + properties: + insecure_skip_verify: + type: boolean + default: false + client_cert: + type: string + minLength: 1 + client_key: + type: string + minLength: 1 + ca_cert: + type: string + minLength: 1 + required: + - client_cert + - client_key + - ca_cert + batch: + type: object + additionalProperties: false + description: Batch/consumer tuning passed to the Kafka reader. + properties: + min_bytes: + type: integer + minimum: 1 + max_bytes: + type: integer + minimum: 1 + max_wait: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + queue_size: + type: integer + minimum: 1 + commit_interval: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" +required: + - source + - brokers + - topic +examples: + - source: kafka + brokers: + - localhost:9092 + topic: crowdsec + group_id: crowdsec-consumers + labels: + type: kafka diff --git a/pkg/acquisition/schemas/kinesis.yaml b/pkg/acquisition/schemas/kinesis.yaml new file mode 100644 index 00000000000..231df491b40 --- /dev/null +++ b/pkg/acquisition/schemas/kinesis.yaml @@ -0,0 +1,113 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Kinesis datasource +description: > + Schema for kinesis acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/kinesis.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: kinesis + description: > + Must be kinesis to bind this acquisition entry to the Kinesis + datasource. + mode: + type: string + enum: [tail] + default: tail + description: > + Acquisition mode (only tail streaming is supported). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: kinesis). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + stream_name: + type: string + description: > + Kinesis stream name. Mandatory when use_enhanced_fanout is false; + mutually exclusive with stream_arn. + stream_arn: + type: string + description: > + Kinesis stream ARN. Mandatory when use_enhanced_fanout is true; + mutually exclusive with stream_name. + use_enhanced_fanout: + type: boolean + default: false + description: > + Use RegisterStreamConsumer/SubscribeToShard instead of GetRecords + polling. + aws_profile: + type: string + description: Named AWS profile to use for credentials. + aws_region: + type: string + default: us-east-1 + description: AWS region to use. + aws_endpoint: + type: string + description: Override the AWS Kinesis API endpoint (for local testing). + consumer_name: + type: string + description: > + Enhanced fan-out consumer name; mandatory when use_enhanced_fanout is + true. + from_subscription: + type: boolean + default: false + description: Internal flag set when reading via a shard subscription. + max_retries: + type: integer + minimum: 1 + default: 10 + description: Maximum number of retries for Kinesis API calls. +required: + - source +allOf: + - not: + required: [stream_name, stream_arn] + - if: + properties: + use_enhanced_fanout: + const: true + required: [use_enhanced_fanout] + then: + required: [stream_arn, consumer_name] + else: + required: [stream_name] +examples: + - source: kinesis + stream_name: my-stream + aws_region: eu-west-1 + labels: + type: kinesis diff --git a/pkg/acquisition/schemas/loki.yaml b/pkg/acquisition/schemas/loki.yaml new file mode 100644 index 00000000000..b6dafec5002 --- /dev/null +++ b/pkg/acquisition/schemas/loki.yaml @@ -0,0 +1,121 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Loki datasource +description: > + Schema for loki acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/loki.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: loki + description: > + Must be loki to bind this acquisition entry to the Loki datasource. + mode: + type: string + enum: [tail, cat] + default: tail + description: > + Acquisition mode (tail streams new entries, cat performs a finite + read). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: loki). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + url: + type: string + minLength: 1 + description: Base URL of the Loki server. + prefix: + type: string + default: "/" + description: > + Path prefix for the Loki API, normalized to end with a slash. + query: + type: string + minLength: 1 + description: LogQL query used to select streams. Mandatory. + limit: + type: integer + minimum: 1 + default: 100 + description: Maximum number of log lines fetched per request. + delay_for: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 0s + description: > + Delay applied before querying, between 0s and 5s. + since: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: > + Lower time bound for the query; reset to 0 in tail mode. + headers: + type: object + additionalProperties: + type: string + description: Extra HTTP headers sent to Loki. + wait_for_ready: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 10s + description: Retry interval while waiting for Loki readiness. + auth: + type: object + additionalProperties: false + description: Basic auth credentials for Loki. + properties: + username: + type: string + password: + type: string + max_failure_duration: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 30s + description: > + Maximum duration of consecutive query failures before the source + stops. + no_ready_check: + type: boolean + default: false + description: Bypass the /ready check before starting. +required: + - source + - query +examples: + - source: loki + url: http://localhost:3100 + query: '{job="varlogs"}' + labels: + type: loki diff --git a/pkg/acquisition/schemas/s3.yaml b/pkg/acquisition/schemas/s3.yaml new file mode 100644 index 00000000000..99b5e4f71bd --- /dev/null +++ b/pkg/acquisition/schemas/s3.yaml @@ -0,0 +1,129 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec S3 datasource +description: > + Schema for s3 acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/s3.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: s3 + description: > + Must be s3 to bind this acquisition entry to the S3 datasource. + mode: + type: string + enum: [tail, cat] + default: tail + description: > + Acquisition mode (tail polls for new objects, cat performs a finite + read). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: s3). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events; strongly recommended for S3 since logs are not + processed in real time. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + aws_profile: + type: string + description: Named AWS profile to use for credentials. + aws_region: + type: string + default: us-east-1 + description: AWS region to use. + aws_endpoint: + type: string + description: Override the AWS S3/SQS API endpoint (for local testing). + bucket_name: + type: string + description: > + S3 bucket to poll. Required when polling_method is list; mutually + exclusive with sqs_name. + prefix: + type: string + description: Restrict listing to keys under this prefix. + polling_method: + type: string + enum: [list, sqs] + default: list + description: > + How new objects are discovered: list (periodic bucket listing) or sqs + (event notifications via an SQS queue). + polling_interval: + type: integer + minimum: 1 + default: 60 + description: > + Interval, in seconds, between bucket listings when polling_method is + list. + sqs_name: + type: string + description: > + SQS queue name to consume object-created notifications from. Required + when polling_method is sqs; mutually exclusive with bucket_name. + sqs_format: + type: string + enum: [eventbridge, s3notification, sns] + description: > + Expected SQS message envelope format. Auto-detected when omitted. + max_buffer_size: + type: integer + minimum: 1 + description: > + Maximum size, in bytes, of a scanned line; defaults to bufio's + MaxScanTokenSize when unset. +required: + - source +allOf: + - not: + required: [bucket_name, sqs_name] + - if: + properties: + polling_method: + const: sqs + required: [polling_method] + then: + required: [sqs_name] + - if: + not: + properties: + polling_method: + const: sqs + required: [polling_method] + then: + required: [bucket_name] +examples: + - source: s3 + bucket_name: my-waf-logs + aws_region: eu-west-1 + use_time_machine: true + labels: + type: s3 diff --git a/pkg/acquisition/schemas/syslog.yaml b/pkg/acquisition/schemas/syslog.yaml new file mode 100644 index 00000000000..f0900f5b0fa --- /dev/null +++ b/pkg/acquisition/schemas/syslog.yaml @@ -0,0 +1,88 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Syslog datasource +description: > + Schema for syslog acquisition entries consumed by CrowdSec. Every field + mirrors pkg/acquisition/modules/syslog.Configuration and the embedded + configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: syslog + description: > + Must be syslog to bind this acquisition entry to the Syslog datasource. + mode: + type: string + enum: [tail] + default: tail + description: > + Acquisition mode (only tail streaming is supported). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: syslog). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + protocol: + type: string + enum: [udp, tcp] + description: Transport protocol to listen on. + listen_addr: + type: string + default: 127.0.0.1 + description: IPv4 or IPv6 address to listen on. + anyOf: + - format: ipv4 + - format: ipv6 + listen_port: + type: integer + minimum: 1 + maximum: 65535 + default: 514 + description: Port to listen on. + max_message_len: + type: integer + minimum: 1 + default: 2048 + description: Maximum accepted syslog message length, in bytes. + disable_rfc_parser: + type: boolean + default: false + description: > + Skip RFC3164/RFC5424 parsing and only strip the PRI header. +required: + - source +examples: + - source: syslog + listen_addr: 0.0.0.0 + listen_port: 514 + protocol: udp + labels: + type: syslog diff --git a/pkg/acquisition/schemas/victorialogs.yaml b/pkg/acquisition/schemas/victorialogs.yaml new file mode 100644 index 00000000000..b9a576e1d7d --- /dev/null +++ b/pkg/acquisition/schemas/victorialogs.yaml @@ -0,0 +1,113 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec VictoriaLogs datasource +description: > + Schema for victorialogs acquisition entries consumed by CrowdSec. Every + field mirrors pkg/acquisition/modules/victorialogs.Configuration and the + embedded configuration.DataSourceCommonCfg. +type: object +additionalProperties: false +properties: + source: + type: string + const: victorialogs + description: > + Must be victorialogs to bind this acquisition entry to the + VictoriaLogs datasource. + mode: + type: string + enum: [tail, cat] + default: tail + description: > + Acquisition mode (tail streams new entries, cat performs a finite + read). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: victorialogs). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + url: + type: string + minLength: 1 + description: Base URL of the VictoriaLogs server. Mandatory. + prefix: + type: string + default: "/" + description: > + Path prefix for the VictoriaLogs API, normalized to end with a slash. + query: + type: string + minLength: 1 + description: LogsQL query used to select entries. Mandatory. + limit: + type: integer + minimum: 1 + default: 100 + description: Maximum number of log lines fetched per request. + since: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + description: > + Lower time bound for the query; reset to 0 in tail mode. + headers: + type: object + additionalProperties: + type: string + description: Extra HTTP headers sent to VictoriaLogs. + wait_for_ready: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 10s + description: Retry interval while waiting for VictoriaLogs readiness. + auth: + type: object + additionalProperties: false + description: Basic auth credentials for VictoriaLogs. + properties: + username: + type: string + password: + type: string + max_failure_duration: + type: string + pattern: "^[0-9]+(ns|us|ms|s|m|h)$" + default: 30s + description: > + Maximum duration of consecutive query failures before the source + stops. +required: + - source + - url + - query +examples: + - source: victorialogs + url: http://localhost:9428 + query: '{job="varlogs"}' + labels: + type: victorialogs diff --git a/pkg/acquisition/schemas/wineventlog.yaml b/pkg/acquisition/schemas/wineventlog.yaml new file mode 100644 index 00000000000..58c6d0953d8 --- /dev/null +++ b/pkg/acquisition/schemas/wineventlog.yaml @@ -0,0 +1,97 @@ +$schema: https://json-schema.org/draft/2020-12/schema +title: CrowdSec Windows Event Log datasource +description: > + Schema for wineventlog acquisition entries consumed by CrowdSec. Every + field mirrors pkg/acquisition/modules/wineventlog.Configuration and the + embedded configuration.DataSourceCommonCfg. Windows-only datasource. +type: object +additionalProperties: false +properties: + source: + type: string + const: wineventlog + description: > + Must be wineventlog to bind this acquisition entry to the Windows + Event Log datasource. + mode: + type: string + enum: [tail] + default: tail + description: > + Acquisition mode (only tail streaming is supported; forced regardless + of the configured value). + labels: + type: object + minProperties: 1 + description: > + Labels attached to emitted events (for example type: wineventlog). + additionalProperties: + type: string + properties: + type: + type: string + description: Parser/collection selector; strongly recommended. + log_level: + type: string + enum: [panic, fatal, error, warn, warning, info, debug, trace] + description: > + Overrides the module logger level for this datasource. + name: + type: string + description: Friendly identifier for the datasource entry. + use_time_machine: + type: boolean + default: false + description: > + Replays past events when supported by the acquisition module. + unique_id: + type: string + description: > + Stable identifier injected by cscli/crowdsec auto-run (usually not user set). + transform: + type: string + description: > + expr program applied to events before they enter the pipeline. + event_channel: + type: string + minLength: 1 + description: > + Windows event channel to subscribe to (for example Security). + Mutually exclusive with xpath_query; one of the two is required. + event_level: + type: string + enum: [CRITICAL, ERROR, WARNING, INFORMATION, VERBOSE] + description: > + Minimum event level to match, used to build the XPath query when + xpath_query is not set. Case-insensitive. + event_ids: + type: array + items: + type: integer + description: > + Event IDs to match, used to build the XPath query when xpath_query is + not set. + xpath_query: + type: string + minLength: 1 + description: > + Raw XPath query to subscribe with. Mutually exclusive with + event_channel; one of the two is required. + pretty_name: + type: string + description: > + Friendly name used in logs in place of the generated query. +required: + - source +allOf: + - not: + required: [event_channel, xpath_query] + - anyOf: + - required: [event_channel] + - required: [xpath_query] +examples: + - source: wineventlog + event_channel: Security + event_ids: [4625] + labels: + type: wineventlog