From 4f6c66edbba85154939fbc0a2ffb6bc20ac84db1 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Fri, 13 Dec 2024 14:41:04 +0000 Subject: [PATCH 1/2] remove modules doc --- docs/index.asciidoc | 5 - docs/static/arcsight-module.asciidoc | 476 ---------------- docs/static/azure-module.asciidoc | 532 ------------------ docs/static/deploying.asciidoc | 4 +- docs/static/modules.asciidoc | 209 ------- docs/static/netflow-module.asciidoc | 195 ------- .../running-logstash-command-line.asciidoc | 16 - docs/static/settings-file.asciidoc | 22 - 8 files changed, 1 insertion(+), 1458 deletions(-) delete mode 100644 docs/static/arcsight-module.asciidoc delete mode 100644 docs/static/azure-module.asciidoc delete mode 100644 docs/static/modules.asciidoc delete mode 100644 docs/static/netflow-module.asciidoc diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 87c8040e7ec..6c92fce5a75 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -137,11 +137,6 @@ include::static/management/configuring-centralized-pipelines.asciidoc[] // (Planting near module content for now. Will likely move it up in info architecture.) include::static/ea-integrations.asciidoc[] -// Working with Logstash Modules -include::static/modules.asciidoc[] - -include::static/arcsight-module.asciidoc[] - // Working with Filebeat Modules include::static/filebeat-modules.asciidoc[] diff --git a/docs/static/arcsight-module.asciidoc b/docs/static/arcsight-module.asciidoc deleted file mode 100644 index 7e985f27270..00000000000 --- a/docs/static/arcsight-module.asciidoc +++ /dev/null @@ -1,476 +0,0 @@ -[role="xpack"] -[[arcsight-module]] -=== Logstash ArcSight Module - -++++ -ArcSight Module (deprecated) -++++ - -deprecated[8.16.0, Replace by https://docs.elastic.co/integrations/cef[Common Event Format (CEF)] from Elastic {integrations}] - -NOTE: The Logstash ArcSight module is an -https://www.elastic.co/products/x-pack[{xpack}] feature under the Basic License -and is therefore free to use. Please contact -mailto:arcsight@elastic.co[arcsight@elastic.co] for questions or more -information. - -The Logstash ArcSight module enables you to easily integrate your ArcSight data with the Elastic Stack. -With a single command, the module taps directly into the ArcSight Smart Connector or the Event Broker, -parses and indexes the security events into Elasticsearch, and installs a suite of Kibana dashboards -to get you exploring your data immediately. - -NOTE: The {ls} ArsSight Module has been deprecated and replaced by https://docs.elastic.co/integrations/cef[Common Event Format (CEF)] from Elastic {integrations}. - -[[arcsight-prereqs]] -==== Prerequisites - -These instructions assume that Logstash, Elasticsearch, and Kibana are already -installed. The products you need are https://www.elastic.co/downloads[available -to download] and easy to install. The Elastic Stack 5.6 (or later) and {xpack} are required for -this module. If you are using the Elastic Stack 6.2 and earlier, please see -the https://www.elastic.co/guide/en/logstash/6.2/arcsight-module.html[instructions] -for those versions. - -[[arcsight-architecture]] -==== Deployment Architecture - -The Logstash ArcSight module understands CEF (Common Event Format), and can -accept, enrich, and index these events for analysis on the Elastic Stack. ADP -contains two core data collection components for data streaming: - -* The _Smart Connectors (SC)_ are edge log collectors that parse and normalize -data to CEF prior to publishing to the Logstash receiver. -* The _Event Broker_ is the central hub for incoming data and is based on -open source Apache Kafka. The Logstash ArcSight module can consume directly from -Event Broker topics. - -[[arcsight-getting-started-smartconnector]] -==== Getting Started with the Smart Connector - -:smart_connect_host: smart_connect_host -:smart_connect_port: smart_connect_port - -To get started, you can use a basic Elastic Stack setup that reads events from -the Smart Connector directly. - -image::static/images/arcsight-diagram-smart-connectors.svg[ArcSight Smart Connector architecture] - -Smart Connector has been configured to publish ArcSight data (to TCP port `5000`) using the CEF syslog -destination. - -NOTE: Logstash, Elasticsearch, and Kibana must run locally. Note that you can also run -Elasticsearch, Kibana and Logstash on separate hosts to consume data from ArcSight. - -[[arcsight-instructions-smartconnector]] -===== Instructions for Smart Connector - -. Start the Logstash ArcSight module by running the following command in the -Logstash install directory with your respective Smart Connector host and port: -+ -["source","shell",subs="attributes"] ------ -bin/logstash --modules arcsight --setup \ - -M "arcsight.var.input.smartconnector.port={smart_connect_port}" \ - -M "arcsight.var.elasticsearch.hosts=localhost:9200" \ - -M "arcsight.var.kibana.host=localhost:5601" ------ -+ --- - -The `--modules arcsight` option spins up an ArcSight CEF-aware Logstash -pipeline for ingestion. The `--setup` option creates an `arcsight-*` index -pattern in Elasticsearch and imports Kibana dashboards and visualizations. On -subsequent module runs or when scaling out the Logstash deployment, -the `--setup` option should be omitted to avoid overwriting the existing Kibana -dashboards. - -See <> for more info. --- - -. Explore your data with Kibana: -.. Open browser @ http://localhost:5601[http://localhost:5601] (username: - "elastic"; password: "{pwd}") -.. Open the *[ArcSight] Network Overview Dashboard* -.. See <> for additional details on data exploration. - -See <> if you want to specify additional options that -control the behavior of the ArcSight module. - -[[arcsight-getting-started-eventbroker]] -==== Getting Started with the Event Broker - -:event_broker_host: event_broker_host -:event_broker_port: event_broker_port - -To get started, you can use a basic Elastic Stack setup that reads events from -the Event Broker event stream. - -image::static/images/arcsight-diagram-adp.svg[ArcSight Event Broker architecture] - -By default, the Logstash ArcSight module consumes from the Event Broker "eb-cef" topic. -For additional settings, see <>. Consuming from a -secured Event Broker port is possible, see <>. - -NOTE: Logstash, Elasticsearch, and Kibana must run locally. Note that you can also run -Elasticsearch, Kibana and Logstash on separate hosts to consume data from ArcSight. - -[[arcsight-instructions-eventbroker]] -===== Instructions for Event Broker - -. Start the Logstash ArcSight module by running the following command in the -Logstash install directory with your respective Event Broker host and port: -+ -["source","shell",subs="attributes"] ------ -bin/logstash --modules arcsight --setup \ - -M "arcsight.var.input.eventbroker.bootstrap_servers={event_broker_host}:{event_broker_port}" \ - -M "arcsight.var.elasticsearch.hosts=localhost:9200" \ - -M "arcsight.var.kibana.host=localhost:5601" ------ -+ --- -The `--modules arcsight` option spins up an ArcSight CEF-aware Logstash -pipeline for ingestion. The `--setup` option creates an `arcsight-*` index -pattern in Elasticsearch and imports Kibana dashboards and visualizations. On -subsequent module runs or when scaling out the Logstash deployment, -the `--setup` option should be omitted to avoid overwriting the existing Kibana -dashboards. - -See <> for more info. --- - -. Explore your data with Kibana: -.. Open browser @ http://localhost:5601[http://localhost:5601] (username: - "elastic"; password: "{pwd}") -.. Open the *[ArcSight] Network Overview Dashboard* -.. See <> for additional details on data exploration. - -See <> if you want to specify additional options that -control the behavior of the ArcSight module. - -[[exploring-data-arcsight]] -==== Exploring Your Security Data -Once the Logstash ArcSight module starts receiving events, you can immediately -begin using the packaged Kibana dashboards to explore and visualize your -security data. The dashboards rapidly accelerate the time and effort required -for security analysts and operators to gain situational and behavioral insights -on network, endpoint, and DNS events flowing through the environment. You can -use the dashboards as-is, or tailor them to work better with existing use cases -and business requirements. - -The dashboards have a navigation pane for context switching and drill downs -across three core use cases: - -* *Network Data* -** Dashboards: Network Overview, Network Suspicious Activity -** Data Types: Network firewalls, intrusion systems, VPN devices - -* *Endpoint Data* -** Dashboards: Endpoint Overview, Endpoint OS Activity -** Data Types: Operating systems, applications, host intrusion systems - -* *DNS Data* -** Dashboards: Microsoft DNS Overview -** Data Types: Microsoft DNS devices - -[[network-dashboards-arsight]] -===== Example Network Dashboards - -[role="screenshot"] -image::static/images/arcsight-network-overview.png[Network overview dashboard] - -[role="screenshot"] -image::static/images/arcsight-network-suspicious.png[Network suspicious activity dashboard] - -These Kibana visualizations enable you to quickly understand the top devices, -endpoints, attackers, and targets. This insight, along with the ability to -instantly drill down on a particular host, port, device, or time range, offers a -holistic view across the entire environment to identify specific segments that -may require immediate attention or action. You can easily discover answers to -questions like: - -* Who are my attackers and what are they targeting? -* Which of my devices or endpoints are the busiest and what services were -rendered? -* How many unique attackers, techniques, signatures, or targets were triggered -at any given point in time? -* What are the top sources, destinations, protocols, and behaviors that are -causing the elevated count of failures? - -[[configuring-arcsight]] -==== Configuring the Module - -You can specify additional options for the Logstash ArcSight module in the -`logstash.yml` configuration file or with overrides through the command line -like in the getting started. For more information about configuring modules, see -<>. - -As an example, the following settings can be appended to `logstash.yml` to -configure your module: - -["source","yaml",subs="attributes"] ------ -modules: - - name: arcsight - var.input.eventbroker.bootstrap_servers: "eb_host:39092" - var.input.eventbroker.topics: "eb_topic" - var.elasticsearch.hosts: "localhost:9200" - var.elasticsearch.username: "elastic" - var.elasticsearch.password: "{pwd}" - var.kibana.host: “localhost:5601” - var.kibana.username: "elastic" - var.kibana.password: "{pwd}" ------ - -[[arcsight-module-config]] -===== Logstash ArcSight Module Configuration Options - -The ArcSight module provides the following settings for configuring the behavior -of the module. These settings include ArcSight-specific options plus common -options that are supported by all Logstash modules. - -When you override a setting at the command line, remember to prefix the setting -with the module name, for example, `arcsight.var.inputs` instead of `var.inputs`. - -If you don't specify configuration settings, Logstash uses the defaults. - -*ArcSight Module Options* - -*`var.inputs`*:: -+ --- -* Value type is <> -* Default value is "eventbroker" --- -+ -Set the input(s) to expose for the Logstash ArcSight module. Valid settings are -"eventbroker", "smartconnector", or "eventbroker,smartconnector" (exposes both -inputs concurrently). - -*ArcSight Module Event Broker specific Options* - -*`var.input.eventbroker.bootstrap_servers`*:: -+ --- -* Value type is <> -* Default value is "localhost:39092" --- -+ -A list of Event Broker URLs to use for establishing the initial connection to the cluster. -This list should be in the form of `host1:port1,host2:port2`. These URLs are -just used for the initial connection to discover the full cluster membership -(which may change dynamically). This list need not contain the full set of -servers. (You may want more than one in case a server is down.) - -*`var.input.eventbroker.topics`*:: -+ --- -* Value type is <> -* Default value is ["eb-cef"] --- -+ -A list of Event Broker topics to subscribe to. - -*`var.input.eventbroker.security_protocol`*:: -+ --- -* Value can be any of: `PLAINTEXT`, `SSL`, `SASL_PLAINTEXT`, `SASL_SSL` -* Default value is `"PLAINTEXT"` --- -+ -Security protocol to use, which can be either of PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL. If you specify anything other than PLAINTEXT then you need to also specify some of the options listed below. When specifying `SSL` or `SASL_SSL` you should supply values for the options prefixed with `ssl_`, when specifying `SASL_PLAINTEXT` or `SASL_SSL` you should supply values for `jaas_path`, `kerberos_config`, `sasl_mechanism` and `sasl_kerberos_service_name`. - -*`var.input.eventbroker.ssl_key_password`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -The password of the private key in the key store file. - -*`var.input.eventbroker.ssl_keystore_location`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -If client authentication is required, this setting stores the keystore path. - -*`var.input.eventbroker.ssl_keystore_password`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -If client authentication is required, this setting stores the keystore password. - -*`var.input.eventbroker.ssl_keystore_type`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -The keystore type. - -*`var.input.eventbroker.ssl_truststore_location`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -The JKS truststore path to validate the Kafka broker's certificate. - -*`var.input.eventbroker.ssl_truststore_password`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -The truststore password. - -*`var.input.eventbroker.ssl_truststore_type`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -The truststore type. - -*`var.input.eventbroker.sasl_kerberos_service_name`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -The Kerberos principal name that Kafka broker runs as. -This can be defined either in Kafka's JAAS config or in Kafka's config. - -*`var.input.eventbroker.sasl_mechanism`*:: -+ --- -* Value type is <> -* Default value is `"GSSAPI"` --- -+ -http://kafka.apache.org/documentation.html#security_sasl[SASL mechanism] used for client connections. -This may be any mechanism for which a security provider is available. -GSSAPI is the default mechanism. - -*`var.input.eventbroker.jaas_path`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -The Java Authentication and Authorization Service (JAAS) API supplies user authentication and authorization -services for Kafka. This setting provides the path to the JAAS file. Sample JAAS file for Kafka client: -+ -[source,java] ----------------------------------- -KafkaClient { - com.sun.security.auth.module.Krb5LoginModule required - useTicketCache=true - renewTicket=true - serviceName="kafka"; - }; ----------------------------------- -+ - -Please note that specifying `jaas_path` and `kerberos_config` here will add these -to the global JVM system properties. This means if you have multiple Kafka inputs, -all of them would be sharing the same `jaas_path` and `kerberos_config`. -If this is not desirable, you would have to run separate instances of Logstash on -different JVM instances. - - -*`var.input.eventbroker.kerberos_config`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -Optional path to kerberos config file. This is krb5.conf style as detailed in https://web.mit.edu/kerberos/krb5-1.12/doc/admin/conf_files/krb5_conf.html - -*ArcSight Module Smart Connector specific Options* - -*`var.input.smartconnector.port`*:: -+ --- -* Value type is <> -* Default value is 5000 --- -+ -The TCP port to listen on when receiving data from SCs. - -*`var.input.smartconnector.ssl_enable`*:: -+ --- -* Value type is <> -* Default value is `false` --- -+ -Enable SSL (must be set for other `ssl_` options to take effect). - -*`var.input.smartconnector.ssl_cert`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -SSL certificate path. - -*`var.input.smartconnector.ssl_extra_chain_certs`*:: -+ --- -* Value type is <> -* Default value is `[]` --- -+ -An Array of paths to extra X509 certificates to be added to the certificate chain. -Useful when the CA chain is not necessary in the system store. - -*`var.input.smartconnector.ssl_key`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -SSL key path - -*`var.input.smartconnector.ssl_key_passphrase`*:: -+ --- -* Value type is <> -* Default value is `nil` --- -+ -SSL key passphrase - -*`var.input.smartconnector.ssl_verify`*:: -+ --- -* Value type is <> -* Default value is `true` --- -+ -Verify the identity of the other end of the SSL connection against the CA. -For input, sets the field `sslsubject` to that of the client certificate. - -:smart_connect_host!: -:smart_connect_port!: -:event_broker_host!: -:event_broker_port!: - -include::shared-module-options.asciidoc[] diff --git a/docs/static/azure-module.asciidoc b/docs/static/azure-module.asciidoc deleted file mode 100644 index 14c9caea3a8..00000000000 --- a/docs/static/azure-module.asciidoc +++ /dev/null @@ -1,532 +0,0 @@ -[role="xpack"] -[[azure-module]] -=== Azure Module -experimental[] - -++++ -Azure Module (deprecated) -++++ - -deprecated[7.8.0, "We recommend using the Azure modules in {filebeat-ref}/filebeat-module-azure.html[{Filebeat}] and {metricbeat-ref}/metricbeat-module-azure.html[{metricbeat}], which are compliant with the {ecs-ref}/index.html[Elastic Common Schema (ECS)]"] - -The https://azure.microsoft.com/en-us/overview/what-is-azure/[Microsoft Azure] -module in Logstash helps you easily integrate your Azure activity logs and SQL -diagnostic logs with the Elastic Stack. - -image::static/images/azure-flow.png["Azure Work Flow",width="80%"] - -You can monitor your Azure cloud environments and SQL DB deployments with -deep operational insights across multiple Azure subscriptions. You can explore -the health of your infrastructure in real-time, accelerating root cause analysis -and decreasing overall time to resolution. The Azure module helps you: - -* Analyze infrastructure changes and authorization activity -* Identify suspicious behaviors and potential malicious actors -* Perform root-cause analysis by investigating user activity -* Monitor and optimize your SQL DB deployments. - -The Azure module uses the -{logstash-ref}/plugins-inputs-azure_event_hubs.html[Logstash Azure Event Hubs -input plugin] to consume data from Azure Event Hubs. The module taps directly into the -Azure dashboard, parses and indexes events into Elasticsearch, and installs a -suite of {kib} dashboards to help you start exploring your data immediately. - -[[azure-dashboards]] -==== Dashboards - -These {kib} dashboards are available and ready for you to use. You can use them as they are, or tailor them to meet your needs. - -===== Infrastructure activity monitoring - -* *Overview*. Top-level view into your Azure operations, including info about users, resource groups, service health, access, activities, and alerts. - -* *Alerts*. Alert info, including activity, alert status, and alerts heatmap - -* *User Activity*. Info about system users, their activity, and requests. - -===== SQL Database monitoring - -* *SQL DB Overview*. Top-level view into your SQL Databases, including counts for databases, servers, resource groups, and subscriptions. - -* *SQL DB Database View*. Detailed info about each SQL Database, including wait time, errors, DTU and storage utilization, size, and read and write input/output. - -* *SQL DB Queries*. Info about SQL Database queries and performance. - -[[azure-module-prereqs]] -==== Prerequisites - -Azure Monitor enabled with Azure Event Hubs and the Elastic Stack are required -for this module. - -[[azure-elk-prereqs]] -===== Elastic prerequisites - -The instructions below assume that you have {ls}, {es}, and {kib} running locally. -You can also run {ls}, {es}, and {kib} on separate hosts. - -The Elastic Stack version 6.4 (or later) is required for this module. - -The Azure module uses the `azure_event_hubs` input plugin to consume logs and -metrics from your Azure environment. It is installed by default with {ls} 6.4 -(or later). Basic understanding of the plugin and options is helpful when you -set up the Azure module. -See the {logstash-ref}/plugins-inputs-azure_event_hubs.html[azure_event_hubs input -plugin documentation] for more information. - -Elastic products are https://www.elastic.co/downloads[available to download] and -easy to install. - -[[azure-prereqs]] -===== Azure prerequisites - -Azure Monitor should be configured to stream logs to one or more Event Hubs. -Logstash will need to access these Event Hubs instances to consume your Azure logs and metrics. -See <> at the end of this topic for links to Microsoft Azure documentation. - -[[configuring-azure]] -==== Configure the module - -Specify <> for the Logstash Azure module in the -`logstash.yml` configuration file. - -* *Basic configuration.* You can use the `logstash.yml` file to configure inputs from multiple Event Hubs that share the same configuration. -Basic configuration is recommended for most use cases. - -* *Advanced configuration.* The advanced configuration is available for deployments where different Event Hubs -require different configurations. The `logstash.yml` file holds your settings. Advanced configuration is not necessary or -recommended for most use cases. - -See the {logstash-ref}/plugins-inputs-azure_event_hubs.html[azure_event_hubs input plugin -documentation] for more information about basic and advanced configuration -models. - -[[basic-config-sample]] -===== Basic configuration sample - -The configuration in the `logstash.yml` file is shared between Event Hubs. -Basic configuration is recommended for most use cases - -["source","shell",subs="attributes"] ------ -modules: - - name: azure - var.elasticsearch.hosts: localhost:9200 - var.kibana.host: localhost:5601 - var.input.azure_event_hubs.consumer_group: "logstash" <1> - var.input.azure_event_hubs.storage_connection: "DefaultEndpointsProtocol=https;AccountName=instance1..." <2> - var.input.azure_event_hubs.threads: 9 <3> - var.input.azure_event_hubs.event_hub_connections: - - "Endpoint=sb://...EntityPath=insights-operational-logs" <4> - - "Endpoint=sb://...EntityPath=insights-metrics-pt1m" <5> - - "Endpoint=sb://...EntityPath=insights-logs-blocks" - - "Endpoint=sb://...EntityPath=insights-logs-databasewaitstatistics" - - "Endpoint=sb://...EntityPath=insights-logs-errors" - - "Endpoint=sb://...EntityPath=insights-logs-querystoreruntimestatistics" - - "Endpoint=sb://...EntityPath=insights-logs-querystorewaitstatistics" - - "Endpoint=sb://...EntityPath=insights-logs-timeouts" ------ -<1> The `consumer_group` (optional) is highly recommended. See <>. -<2> The `storage_connection` (optional) sets the Azure Blob Storage connection for tracking processing state for Event Hubs when scaling out a deployment with multiple Logstash instances. See <> for additional details. -<3> See <> for guidelines on choosing an appropriate number of threads. -<4> This connection sets up the consumption of Activity Logs. By default, Azure Monitor uses the `insights-operational-logs` Event Hub name. Make sure this matches the name of the Event Hub specified for Activity Logs. -<5> This connection and the ones below set up the consumption of SQL DB diagnostic logs and metrics. By default, Azure Monitor uses all these different Event Hub names. - -The basic configuration requires the `var.input.azure_event_hubs.` prefix -before a configuration option. -Notice the notation for the `threads` option. - -[[adv-config-sample]] -===== Advanced configuration sample - -Advanced configuration in the `logstash.yml` file supports Event Hub specific -options. Advanced configuration is available for more granular tuning of -threading and Blob Storage usage across multiple Event Hubs. Advanced -configuration is not necessary or recommended for most use cases. Use it only if -it is required for your deployment scenario. - -You must define the `header` array with `name` in the first position. You can -define other options in any order. The per Event Hub configuration takes -precedence. Any values not defined per Event Hub use the global config value. - -In this example `threads`, `consumer_group`, and `storage_connection` will be -applied to each of the configured Event Hubs. Note that `decorate_events` is -defined in both the global and per Event Hub configuration. The per Event Hub -configuration takes precedence, and the global configuration is effectively -ignored when the per Event Hub setting is present. - -["source","shell",subs="attributes"] ------ -modules: - - name: azure - var.elasticsearch.hosts: localhost:9200 - var.kibana.host: localhost:5601 - var.input.azure_event_hubs.decorate_events: true <1> - var.input.azure_event_hubs.threads: 9 <2> - var.input.azure_event_hubs.consumer_group: "logstash" - var.input.azure_event_hubs.storage_connection: "DefaultEndpointsProtocol=https;AccountName=instance1..." - var.input.azure_event_hubs.event_hubs: - - ["name", "initial_position", "storage_container", "decorate_events", "event_hub_connection"] <3> - - ["insights-operational-logs", "TAIL", "activity-logs1", "true", "Endpoint=sb://...EntityPath=insights-operational-logs"] - - ["insights-operational-logs", "TAIL", "activity_logs2", "true", "Endpoint=sb://...EntityPath=insights-operational-logs"] <4> - - ["insights-metrics-pt1m", "TAIL", "dbmetrics", "true", "Endpoint=sb://...EntityPath=insights-metrics-pt1m"] - - ["insights-logs-blocks", "TAIL", "dbblocks", "true", "Endpoint=sb://...EntityPath=insights-logs-blocks"] - - ["insights-logs-databasewaitstatistics", "TAIL", "dbwaitstats", "false", "Endpoint=sb://...EntityPath=insights-logs-databasewaitstatistics"] - - ["insights-logs-errors", "HEAD", "dberrors", "true", "Endpoint=sb://...EntityPath=insights-logs-errors" - - ["insights-logs-querystoreruntimestatistics", "TAIL", "dbstoreruntime", "true", "Endpoint=sb://...EntityPath=insights-logs-querystoreruntimestatistics"] - - ["insights-logs-querystorewaitstatistics", "TAIL", "dbstorewaitstats", "true", "Endpoint=sb://...EntityPath=insights-logs-querystorewaitstatistics"] - - ["insights-logs-timeouts", "TAIL", "dbtimeouts", "true", "Endpoint=sb://...EntityPath=insights-logs-timeouts"] ------ -<1> You can specify global Event Hub options. They will be overridden by any configurations specified in the event_hubs option. -<2> See <> for guidelines on choosing an appropriate number of threads. -<3> The header array must be defined with name in the first position. Other options can be defined in any order. The per Event Hub configuration takes precedence. Any values not defined per Event Hub use the global config value. -<4> This enables consuming from a second Activity Logs Event Hub that uses a different Blob Storage container. This is necessary to avoid the offsets from the first insights-operational-logs from overwriting the offsets for the second insights-operational-logs. - -The advanced configuration doesn't require a prefix before a per Event Hub -configuration option. Notice the notation for the `initial_position` option. - -[[scaling-blob]] -===== Scale Event Hub consumption - -An https://azure.microsoft.com/en-us/services/storage/blobs[Azure Blob Storage -account] is an essential part of Azure-to-Logstash configuration. -It is required for users who want to scale out multiple {ls} instances to consume from Event Hubs. - -A Blob Storage account is a central location that enables multiple instances of -{ls} to work together to process events. It records the -offset (location) of processed events. On restart, {ls} resumes processing -exactly where it left off. - -Configuration notes: - -* A Blob Storage account is highly recommended for use with this module, and is -likely required for production servers. -* The `storage_connection` option passes the blob storage connection string. -* Configure all {ls} instances to use the same `storage_connection` to get the -benefits of shared processing. - -Sample Blob Storage connection string: - -[source,text] ----- -DefaultEndpointsProtocol=https;AccountName=logstash;AccountKey=ETOPnkd/hDAWidkEpPZDiXffQPku/SZdXhPSLnfqdRTalssdEuPkZwIcouzXjCLb/xPZjzhmHfwRCGo0SBSw==;EndpointSuffix=core.windows.net ----- - -Find the connection string to Blob Storage here: -https://portal.azure.com[Azure Portal]`-> Blob Storage account -> Access keys`. - -[[azure_best_practices]] -===== Best practices - -Here are some guidelines to help you achieve a successful deployment, and avoid -data conflicts that can cause lost events. - -* <> -* <> -* <> - -[[azure-bp-group]] -====== Create a {ls} consumer group - -Create a new consumer group specifically for {ls}. Do not use the $default or -any other consumer group that might already be in use. Reusing consumer groups -among non-related consumers can cause unexpected behavior and possibly lost -events. All {ls} instances should use the same consumer group so that they can -work together for processing events. - -[[azure-bp-multihub]] -====== Avoid overwriting offset with multiple Event Hubs - -The offsets (position) of the Event Hubs are stored in the configured Azure Blob -store. The Azure Blob store uses paths like a file system to store the offsets. -If the paths between multiple Event Hubs overlap, then the offsets may be stored -incorrectly. - -To avoid duplicate file paths, use the advanced configuration model and make -sure that at least one of these options is different per Event Hub: - -* storage_connection -* storage_container (defaults to Event Hub name if not defined) -* consumer_group - -[[azure-bp-threads]] -====== Set number of threads correctly - -By default, the number of threads used to service all event hubs is `16`. And -while this may be sufficient for most use cases, throughput may be improved by -refining this number. When servicing a large number of partitions across one or -more event hubs, setting a higher value may result in improved performance. The -maximum number of threads is not strictly bound by the total number of -partitions being serviced, but setting the value much higher than that may mean -that some threads are idle. - -NOTE: The number of threads *must* be greater than or equal to the number of Event -hubs plus one. - -NOTE: Threads are currently available only as a global setting across all event hubs -in a single `azure_event_hubs` input definition. However if your configuration -includes multiple `azure_event_hubs` inputs, the threads setting applies -independently to each. - -**Sample scenarios:** - -* Event Hubs = 4. Partitions on each Event Hub = 3. -Minimum threads is 5 (4 Event Hubs plus one). Maximum threads is 13 (4 Event -Hubs times 3 partitions plus one). -* If you're collecting activity logs from only one specified event hub instance, -then only 2 threads (1 Event Hub plus one) are required. - -[[azure-module-setup]] -==== Set up and run the module - -Be sure that the `logstash.yml` file is <>. - -===== First time setup - -Run this command from the Logstash directory: - -["source","shell",subs="attributes"] ------ -bin/logstash --setup ------ - -The `--modules azure` option starts a Logstash pipeline for ingestion from Azure -Event Hubs. The `--setup` option creates an `azure-*` index pattern in -Elasticsearch and imports Kibana dashboards and visualizations. - -===== Subsequent starts - -Run this command from the Logstash directory: - -["source","shell",subs="attributes"] ------ -bin/logstash ------ - -NOTE: The `--setup` option is intended only for first-time setup. If you include -`--setup` on subsequent runs, your existing Kibana dashboards will be -overwritten. - -[[exploring-data-azure]] -==== Explore your data -When the Logstash Azure module starts receiving events, you can begin using the -packaged Kibana dashboards to explore and visualize your data. - -To explore your data with Kibana: - -. Open a browser to http://localhost:5601[http://localhost:5601] (username: - "elastic"; password: "{pwd}") -. Click *Dashboard*. -. Click *[Azure Monitor] Overview*. - -[[azure_config_options]] -==== Configuration options - -NOTE: All Event Hubs options are common to both basic and advanced -configurations, with the following exceptions. The basic configuration uses -`event_hub_connections` to support multiple connections. The advanced -configuration uses `event_hubs` and `event_hub_connection` (singular). - -[[azure_event_hubs]] -===== `event_hubs` -* Value type is <> -* No default value -* Ignored for basic and command line configuration -* Required for advanced configuration - -Defines the per Event Hubs configuration for the <>. - -The advanced configuration uses `event_hub_connection` instead of `event_hub_connections`. -The `event_hub_connection` option is defined per Event Hub. - -[[azure_event_hub_connections]] -===== `event_hub_connections` -* Value type is <> -* No default value -* Required for basic and command line configuration -* Ignored for advanced configuration - -List of connection strings that identifies the Event Hubs to be read. Connection -strings include the EntityPath for the Event Hub. - - -[[azure_checkpoint_interval]] -===== `checkpoint_interval` -* Value type is <> -* Default value is `5` seconds -* Set to `0` to disable. - -Interval in seconds to write checkpoints during batch processing. Checkpoints -tell {ls} where to resume processing after a restart. Checkpoints are -automatically written at the end of each batch, regardless of this setting. - -Writing checkpoints too frequently can slow down processing unnecessarily. - - -[[azure_consumer_group]] -===== `consumer_group` -* Value type is <> -* Default value is `$Default` - -Consumer group used to read the Event Hub(s). Create a consumer group -specifically for Logstash. Then ensure that all instances of Logstash use that -consumer group so that they can work together properly. - - -[[azure_decorate_events]] -===== `decorate_events` - -* Value type is <> -* Default value is `false` - -Adds metadata about the Event Hub, including `Event Hub name`, `consumer_group`, -`processor_host`, `partition`, `offset`, `sequence`, `timestamp`, and `event_size`. - - -[[azure_initial_position]] -===== `initial_position` -* Value type is <> -* Valid arguments are `beginning`, `end`, `look_back` -* Default value is `beginning` - -When first reading from an Event Hub, start from this position: - -* `beginning` reads all pre-existing events in the Event Hub -* `end` does not read any pre-existing events in the Event Hub -* `look_back` reads `end` minus a number of seconds worth of pre-existing events. -You control the number of seconds using the `initial_position_look_back` option. - -If `storage_connection` is set, the `initial_position` value is used only -the first time Logstash reads from the Event Hub. - - -[[azure_initial_position_look_back]] -===== `initial_position_look_back` -* Value type is <> -* Default value is `86400` -* Used only if `initial_position` is set to `look-back` - -Number of seconds to look back to find the initial position for pre-existing -events. This option is used only if `initial_position` is set to `look_back`. If -`storage_connection` is set, this configuration applies only the first time {ls} -reads from the Event Hub. - - -[[azure_max_batch_size]] -===== `max_batch_size` - -* Value type is <> -* Default value is `125` - -Maximum number of events retrieved and processed together. A checkpoint is -created after each batch. Increasing this value may help with performance, but -requires more memory. - - -[[azure_storage_connection]] -===== `storage_connection` -* Value type is <> -* No default value - -Connection string for blob account storage. Blob account storage persists the -offsets between restarts, and ensures that multiple instances of Logstash -process different partitions. -When this value is set, restarts resume where processing left off. -When this value is not set, the `initial_position` value is used on every restart. - -We strongly recommend that you define this value for production environments. - - -[[azure_storage_container]] -===== `storage_container` -* Value type is <> -* Defaults to the Event Hub name if not defined - -Name of the storage container used to persist offsets and allow multiple instances of {ls} -to work together. - -To avoid overwriting offsets, you can use different storage containers. This is -particularly important if you are monitoring two Event Hubs with the same name. -You can use the advanced configuration model to configure different storage -containers. - - -[[azure_threads]] -===== `threads` -* Value type is <> -* Minimum value is `2` -* Default value is `16` - -Total number of threads used to process events. The value you set here applies -to all Event Hubs. Even with advanced configuration, this value is a global -setting, and can't be set per event hub. - -The number of threads should be the number of Event Hubs plus one or more. -See <> for more information. - - -include::shared-module-options.asciidoc[] - -==== Azure module schema - -This module reads data from the Azure Event Hub and adds some additional structure to the data for Activity Logs and SQL Diagnostics. The original data is always preserved and any data added or parsed will be namespaced under 'azure'. For example, 'azure.subscription' may have been parsed from a longer more complex URN. - -[cols="<,<,<",options="header",] -|======================================================================= -|Name |Description|Notes - -|azure.subscription |Azure subscription from which this data originates. |Some Activity Log events may not be associated with a subscription. -|azure.group |Primary type of data. |Current values are either 'activity_log' or 'sql_diagnostics' -|azure.category* |Secondary type of data specific to group from which the data originated | -|azure.provider |Azure provider | -|azure.resource_group |Azure resource group | -|azure.resource_type |Azure resource type | -|azure.resource_name |Azure resource name | -|azure.database |Azure database name, for display purposes |SQL Diagnostics only -|azure.db_unique_id |Azure database name that is guaranteed to be unique |SQL Diagnostics only -|azure.server |Azure server for the database |SQL Diagnostics only -|azure.server_and_database |Azure server and database combined |SQL Diagnostics only -|azure.metadata |Any @metadata added by the plugins, for example var.input.azure_event_hubs.decorate_events: true -|======================================================================= - -Notes: - -* Activity Logs can have the following categories: Administrative, ServiceHealth, Alert, Autoscale, Security -* SQL Diagnostics can have the following categories: Metric, Blocks, Errors, Timeouts, QueryStoreRuntimeStatistics, QueryStoreWaitStatistics, DatabaseWaitStatistics, SQLInsights - -Microsoft documents Activity log schema -https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-activity-log-schema[here]. -The SQL Diagnostics data is documented -https://docs.microsoft.com/en-us/azure/sql-database/sql-database-metrics-diag-logging[here]. -Elastic does not own these data models, and as such, cannot make any -assurances of information accuracy or passivity. - -===== Special note - Properties field - -Many of the logs contain a `properties` top level field. This is often where the -most interesting data lives. There is not a fixed schema between log types for -properties fields coming from different sources. - -For example, one log may have -`properties.type` where one log sets this a String type and another sets this an -Integer type. To avoid mapping errors, the original properties field is moved to -`__properties.`. -For example -`properties.type` may end up as `sql_diagnostics_Errors_properties.type` or -`activity_log_Security_properties.type` depending on the group/category where -the event originated. - -[[azure-production]] -==== Deploying the module in production - -Use security best practices to secure your configuration. -See {ref}/secure-cluster.html[Secure a cluster] for details and recommendations. - -[[azure-resources]] -==== Microsoft Azure resources - -Microsoft is the best source for the most up-to-date Azure information. - -* https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-overview-azure-monitor[Overview of Azure Monitor] -* https://docs.microsoft.com/en-us/azure/sql-database/sql-database-metrics-diag-logging[Azure SQL Database metrics and diagnostics logging] -* https://docs.microsoft.com/en-us/azure/monitoring-and-diagnostics/monitoring-stream-activity-logs-event-hubs[Stream the Azure Activity Log to Event Hubs] - diff --git a/docs/static/deploying.asciidoc b/docs/static/deploying.asciidoc index be5d46800dc..d39936b033b 100644 --- a/docs/static/deploying.asciidoc +++ b/docs/static/deploying.asciidoc @@ -180,9 +180,7 @@ with the <>. <>. * SNMP trap - Logstash has a native <>. * CEF - Logstash accepts and parses CEF data from systems like Arcsight -SmartConnectors with the <>. See this -https://www.elastic.co/blog/integrating-elastic-stack-with-arcsight-siem-part-1[blog series] -for more details. +SmartConnectors with the <>. [float] ==== Centralized Syslog Servers diff --git a/docs/static/modules.asciidoc b/docs/static/modules.asciidoc deleted file mode 100644 index 8b596e96a0f..00000000000 --- a/docs/static/modules.asciidoc +++ /dev/null @@ -1,209 +0,0 @@ -[[logstash-modules]] -== Working with Logstash Modules - -Logstash modules provide a quick, end-to-end solution for ingesting data and -visualizing it with purpose-built dashboards. - -These modules are available: - -* <> - -Each module comes pre-packaged with Logstash configurations, Kibana dashboards, -and other meta files that make it easier for you to set up the Elastic Stack for -specific use cases or data sources. - -You can think of modules as providing three essential functions that make it -easier for you to get started. When you run a module, it will: - -. Create the Elasticsearch index. - -. Set up the Kibana dashboards, including the index pattern, searches, and -visualizations required to visualize your data in Kibana. - -. Run the Logstash pipeline with the configurations required to read and parse -the data. - -image::static/images/logstash-module-overview.png[Logstash modules overview] - -[discrete] -[[running-logstash-modules]] -=== Running modules - -To run a module and set up dashboards, you specify the following options: - -[source,shell] ----- -bin/logstash --modules MODULE_NAME --setup [-M "CONFIG_SETTING=VALUE"] ----- - - -//TODO: For 6.0, show how to run multiple modules - -Where: - -* `--modules` runs the Logstash module specified by `MODULE_NAME`. - -* `-M "CONFIG_SETTING=VALUE"` is optional and overrides the specified -configuration setting. You can specify multiple overrides. Each override must -start with `-M`. See <> for more info. - -* `--setup` creates an index pattern in Elasticsearch and imports Kibana -dashboards and visualizations. Running `--setup` is a one-time setup step. Omit -this option for subsequent runs of the module to avoid overwriting existing -Kibana dashboards. - -For example, the following command runs the Netflow module with the default -settings, and sets up the netflow index pattern and dashboards: - -[source,shell] ----- -bin/logstash --modules netflow --setup ----- - -The following command runs the Netflow module and overrides the Elasticsearch -`host` setting. Here it's assumed that you've already run the setup step. - -[source,shell] ----- -bin/logstash --modules netflow -M "netflow.var.elasticsearch.host=es.mycloud.com" ----- - - -[discrete] -[[configuring-logstash-modules]] -=== Configuring modules - -To configure a module, you can either -<> in the -`logstash.yml` <>, or use command-line overrides to -<>. - -[discrete] -[[setting-logstash-module-config]] -==== Specify module settings in `logstash.yml` - -To specify module settings in the `logstash.yml` -<> file, you add a module definition to -the modules array. Each module definition begins with a dash (-) and is followed -by `name: module_name` then a series of name/value pairs that specify module -settings. For example: - -[source,shell] ----- -modules: -- name: netflow - var.elasticsearch.hosts: "es.mycloud.com" - var.elasticsearch.username: "foo" - var.elasticsearch.password: "password" - var.kibana.host: "kb.mycloud.com" - var.kibana.username: "foo" - var.kibana.password: "password" - var.input.tcp.port: 5606 ----- - -For a list of available module settings, see the documentation for the module. - -[discrete] -[[overriding-logstash-module-settings]] -==== Specify module settings at the command line - -[IMPORTANT] -.macOS Gatekeeper warnings -==== -Apple's rollout of stricter notarization requirements affected the notarization -of the {version} {ls} artifacts. If macOS Catalina displays a dialog when you -first run {ls} that interrupts it, you will need to take an action to allow it -to run. -To prevent Gatekeeper checks on the {ls} files, run the following command on the -downloaded `.tar.gz` archive or the directory to which was extracted: -[source,sh] ----- -xattr -d -r com.apple.quarantine ----- -For example, if the `.tar.gz` file was extracted to the default -logstash-{version} directory, the command is: -[source,sh,subs="attributes"] ----- -xattr -d -r com.apple.quarantine logstash-{version} ----- -Alternatively, you can add a security override if a Gatekeeper popup appears by -following the instructions in the _How to open an app that hasn’t been notarized -or is from an unidentified developer_ section of -https://support.apple.com/en-us/HT202491[Safely open apps on your Mac]. -==== - -You can override module settings by specifying one or more configuration -overrides when you start Logstash. To specify an override, you use the `-M` -command line option: - -[source,shell] ----- --M MODULE_NAME.var.PLUGINTYPE1.PLUGINNAME1.KEY1=VALUE ----- - -Notice that the fully-qualified setting name includes the module name. - -You can specify multiple overrides. Each override must start with `-M`. - -The following command runs the Netflow module and overrides both the -Elasticsearch `host` setting and the `udp.port` setting: - -[source,shell] ----- -bin/logstash --modules netflow -M "netflow.var.input.udp.port=3555" -M "netflow.var.elasticsearch.hosts=my-es-cloud" ----- - -Any settings defined in the command line are ephemeral and will not persist across -subsequent runs of Logstash. If you want to persist a configuration, you need to -set it in the `logstash.yml` <>. - -Settings that you specify at the command line are merged with any settings -specified in the `logstash.yml` file. If an option is set in both -places, the value specified at the command line takes precedence. - -[discrete] -[[module-to-cloud]] -==== Sending data to Elasticsearch Service from modules - -Cloud ID and Cloud Auth can be specified in the `logstash.yml` <>. -They should be added separately from any module configuration settings you may have added before. - -Cloud ID overwrites these settings: ----- -var.elasticsearch.hosts -var.kibana.host ----- - -Cloud Auth overwrites these settings: ----- -var.elasticsearch.username -var.elasticsearch.password -var.kibana.username -var.kibana.password ----- - -["source","yaml",subs="attributes"] ----- -# example with a label -cloud.id: "staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy" -cloud.auth: "elastic:{pwd}" ----- - -["source","yaml",subs="attributes"] ----- -# example without a label -cloud.id: "dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy" -cloud.auth: "elastic:{pwd}" ----- - -These settings can be also specified at the command line, like this: - -["source","sh",subs="attributes,callouts"] ----- -bin/logstash --modules netflow -M "netflow.var.input.udp.port=3555" --cloud.id --cloud.auth ----- - -NOTE: When working with modules, use the dot notation to specify cloud.id and -cloud.auth, as indicated in the examples. - -For more info on Cloud ID and Cloud Auth, see <>. diff --git a/docs/static/netflow-module.asciidoc b/docs/static/netflow-module.asciidoc deleted file mode 100644 index b00c4e8f6de..00000000000 --- a/docs/static/netflow-module.asciidoc +++ /dev/null @@ -1,195 +0,0 @@ -[[netflow-module]] -=== Logstash Netflow Module - -++++ -Netflow Module (deprecated) -++++ - -deprecated[7.4.0, Replaced by the {filebeat-ref}/filebeat-module-netflow.html[{Filebeat} Netflow Module] which is compliant with the {ecs-ref}/index.html[Elastic Common Schema (ECS)]] - -The Logstash Netflow module simplifies the collection, normalization, and -visualization of network flow data. With a single command, the module parses -network flow data, indexes the events into Elasticsearch, and installs a suite -of Kibana dashboards to get you exploring your data immediately. - -Logstash modules support Netflow Version 5 and 9. - - -==== What is Flow Data? - -Netflow is a type of data record streamed from capable network devices. It -contains information about connections traversing the device, and includes -source IP addresses and ports, destination IP addresses and ports, types of -service, VLANs, and other information that can be encoded into frame and -protocol headers. With Netflow data, network operators can go beyond monitoring -simply the volume of data crossing their networks. They can understand where the -traffic originated, where it is going, and what services or applications it is -part of. - -[[netflow-requirements]] -===== Requirements - -These instructions assume you have already installed Elastic Stack -(Logstash, Elasticsearch, and Kibana) version 5.6 or higher. The products you -need are https://www.elastic.co/downloads[available to download] and easy to -install. - -[[netflow-getting-started]] -==== Getting Started - -NOTE: The {ls} Netflow Module has been deprecated and replaced by the -{filebeat-ref}/filebeat-module-netflow.html[{Filebeat} Netflow Module], which is -compliant with the {ecs-ref}/index.html[Elastic Common Schema (ECS)]. - -. Start the Logstash Netflow module by running the following command in the -Logstash installation directory: -+ -[source,shell] ------ -bin/logstash --modules netflow --setup -M netflow.var.input.udp.port=NNNN ------ -+ -Where `NNNN` is the UDP port on which Logstash will listen for network traffic -data. If you don't specify a port, Logstash listens on port 2055 by default. -+ -The `--modules netflow` option spins up a Netflow-aware Logstash pipeline -for ingestion. -+ -The `--setup` option creates a `netflow-*` index pattern in Elasticsearch and -imports Kibana dashboards and visualizations. Running `--setup` is a one-time -setup step. Omit this option for subsequent runs of the module to avoid -overwriting existing Kibana dashboards. -+ -The command shown here assumes that you're running Elasticsearch and Kibana on -your localhost. If you're not, you need to specify additional connection -options. See <>. - -. Explore your data in Kibana: -.. Open your browser and navigate to -http://localhost:5601[http://localhost:5601]. If security is enabled, you'll -need to specify the Kibana username and password that you used when you set up -security. -.. Open *Netflow: Network Overview Dashboard*. -.. See <> for additional details on data exploration. - -[[exploring-data-netflow]] -==== Exploring Your Data - -Once the Logstash Netflow module starts processing events, you can immediately -begin using the packaged Kibana dashboards to explore and visualize your -network flow data. - -You can use the dashboards as-is, or tailor them to work better with existing -use cases and business requirements. - -[[network-dashboards-netflow]] -===== Example Dashboards - -On the *Overview* dashboard, you can see a summary of basic traffic data and set -up filters before you drill down to gain deeper insight into the data. - -[role="screenshot"] -image::static/images/netflow-overview.png[Netflow overview dashboard] - -For example, on the *Conversation Partners* dashboard, you can see the source -and destination addresses of the client and server in any conversation. - -[role="screenshot"] -image::static/images/netflow-conversation-partners.png[Netflow conversation partners dashboard] - -On the *Traffic Analysis* dashboard, you can identify high volume conversations -by viewing the traffic volume in bytes. - -[role="screenshot"] -image::static/images/netflow-traffic-analysis.png[Netflow traffic analysis dashboard] - -Then you can go to the *Geo Location* dashboard where you can visualize the -location of destinations and sources on a heat map. - -[role="screenshot"] -image::static/images/netflow-geo-location.png[Netflow geo location dashboard] - - -[[configuring-netflow]] -==== Configuring the Module - -You can further refine the behavior of the Logstash Netflow module by specifying -settings in the `logstash.yml` settings file, or overriding settings at the -command line. - -For example, the following configuration in the `logstash.yml` file sets -Logstash to listen on port 9996 for network traffic data: -[source,yaml] ------ -modules: - - name: netflow - var.input.udp.port: 9996 ------ - -To specify the same settings at the command line, you use: - -[source,shell] ------ -bin/logstash --modules netflow -M netflow.var.input.udp.port=9996 ------ - -For more information about configuring modules, see -<>. - -[[netflow-module-config]] -===== Configuration Options - -The Netflow module provides the following settings for configuring the behavior -of the module. These settings include Netflow-specific options plus common -options that are supported by all Logstash modules. - -When you override a setting at the command line, remember to prefix the setting -with the module name, for example, `netflow.var.input.udp.port` instead of -`var.input.udp.port`. - -If you don't specify configuration settings, Logstash uses the defaults. - -*Netflow Options* - -*`var.input.udp.port:`*:: -+ --- -* Value type is <> -* Default value is 2055. --- -+ -Sets the UDP port on which Logstash listens for network traffic data. Although -2055 is the default for this setting, some devices use ports in the range of -9995 through 9998, with 9996 being the most commonly used alternative. - -*`var.input.udp.workers:`*:: -+ --- -* Value type is <> -* Default value is 2. --- -+ -Number of threads processing packets. - -*`var.input.udp.receive_buffer_bytes:`*:: -+ --- -* Value type is <> -* Default value is 212992. --- -+ -The socket receive buffer size in bytes. -The operating system will use the max allowed value if receive_buffer_bytes is larger than allowed. -Consult your operating system documentation if you need to increase this max allowed value. - -*`var.input.udp.queue_size:`*:: -+ --- -* Value type is <> -* Default value is 2000. --- -+ -This is the number of unprocessed UDP packets you can hold in memory before -packets will start dropping. - -include::shared-module-options.asciidoc[] diff --git a/docs/static/running-logstash-command-line.asciidoc b/docs/static/running-logstash-command-line.asciidoc index 37873ff967c..f4216f5d4fa 100644 --- a/docs/static/running-logstash-command-line.asciidoc +++ b/docs/static/running-logstash-command-line.asciidoc @@ -104,22 +104,6 @@ With this command, Logstash concatenates three config files, `/tmp/one`, `/tmp/t *`--plugin-classloaders`*:: (Beta) Load Java plugins in independent classloaders to isolate their dependencies. -*`--modules`*:: - Launch the named module. Works in conjunction with the `-M` option to assign values to - default variables for the specified module. If `--modules` is used on the command line, - any modules in `logstash.yml` will be ignored, as will any settings there. This flag is - mutually exclusive to the `-f` and `-e` flags. Only one of `-f`, `-e`, or `--modules` may - be specified. Multiple modules can be specified by separating them with a comma, or by - invoking the `--modules` flag multiple times. - -*`-M, --modules.variable`*:: - Assign a value to a configurable option for a module. The format for assigning variables is - `-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.KEY_NAME=value"` for Logstash variables. For other - settings, it will be `-M "MODULE_NAME.KEY_NAME.SUB_KEYNAME=value"`. The `-M` flag can be used - as many times as is necessary. If no `-M` options are specified, then the default value for - that setting will be used. The `-M` flag is only used in conjunction with the `--modules` - flag. It will be ignored if the `--modules` flag is absent. - *`--pipeline.id ID`*:: Sets the ID of pipeline. The default is `main`. diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index 318ec73084f..17fe6973142 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -45,24 +45,6 @@ path: Note that the `${VAR_NAME:default_value}` notation is supported, setting a default batch delay of `50` and a default `path.queue` of `/tmp/queue` in the above example. -Modules may also be specified in the `logstash.yml` file. The modules definition will have -this format: - -[source,yaml] -------------------------------------------------------------------------------------- -modules: - - name: MODULE_NAME1 - var.PLUGIN_TYPE1.PLUGIN_NAME1.KEY1: VALUE - var.PLUGIN_TYPE1.PLUGIN_NAME1.KEY2: VALUE - var.PLUGIN_TYPE2.PLUGIN_NAME2.KEY1: VALUE - var.PLUGIN_TYPE3.PLUGIN_NAME3.KEY1: VALUE - - name: MODULE_NAME2 - var.PLUGIN_TYPE1.PLUGIN_NAME1.KEY1: VALUE - var.PLUGIN_TYPE1.PLUGIN_NAME1.KEY2: VALUE -------------------------------------------------------------------------------------- - -IMPORTANT: If the <> `--modules` is used, any modules defined in the `logstash.yml` file will be ignored. - The `logstash.yml` file includes the following settings. [options="header"] @@ -191,10 +173,6 @@ Current options are: | `none` -| `modules` -| When configured, `modules` must be in the nested YAML structure described above this table. -| _N/A_ - | `queue.type` | The internal queuing model to use for event buffering. Specify `memory` for legacy in-memory based queuing, or `persisted` for disk-based ACKed queueing (<>). | `memory` From 48f789fce66f8ef8a0c81fb5ed4c8c48f673d373 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 17 Dec 2024 15:39:55 +0000 Subject: [PATCH 2/2] remove module common options --- docs/static/shared-module-options.asciidoc | 181 --------------------- 1 file changed, 181 deletions(-) delete mode 100644 docs/static/shared-module-options.asciidoc diff --git a/docs/static/shared-module-options.asciidoc b/docs/static/shared-module-options.asciidoc deleted file mode 100644 index 3e9b2eb12c0..00000000000 --- a/docs/static/shared-module-options.asciidoc +++ /dev/null @@ -1,181 +0,0 @@ -*Common options* - -The following configuration options are supported by all modules: - -*`var.elasticsearch.hosts`*:: -+ --- -* Value type is <> -* Default value is "localhost:9200" --- -+ -Sets the host(s) of the Elasticsearch cluster. For each host, you must specify -the hostname and port. For example, "myhost:9200". If given an <>, -Logstash will load balance requests across the hosts specified in the hosts -parameter. It is important to exclude {ref}/modules-node.html[dedicated master -nodes] from the hosts list to prevent Logstash from sending bulk requests to the -master nodes. So this parameter should only reference either data or client -nodes in Elasticsearch. -+ -Any special characters present in the URLs here MUST be URL escaped! This means # -should be put in as %23 for instance. - -*`var.elasticsearch.username`*:: -+ --- -* Value type is <> -* Default value is "elastic" --- -+ -The username to authenticate to a secure Elasticsearch cluster. - -*`var.elasticsearch.password`*:: -+ --- -* Value type is <> -* Default value is "changeme" --- -+ -The password to authenticate to a secure Elasticsearch cluster. - -*`var.elasticsearch.ssl.enabled`*:: -+ --- -* Value type is <> -* There is no default value for this setting. --- -+ -Enable SSL/TLS secured communication to the Elasticsearch cluster. Leaving this -unspecified will use whatever scheme is specified in the URLs listed in `hosts`. -If no explicit protocol is specified, plain HTTP will be used. If SSL is -explicitly disabled here, the plugin will refuse to start if an HTTPS URL is -given in hosts. - -*`var.elasticsearch.ssl.verification_mode`*:: -+ --- -* Value type is <> -* Default value is "strict" --- -+ -The hostname verification setting when communicating with Elasticsearch. Set to -`disable` to turn off hostname verification. Disabling this has serious security -concerns. - -*`var.elasticsearch.ssl.certificate_authority`*:: -+ --- -* Value type is <> -* There is no default value for this setting --- -+ -The path to an X.509 certificate to use to validate SSL certificates when -communicating with Elasticsearch. - -*`var.elasticsearch.ssl.certificate`*:: -+ --- -* Value type is <> -* There is no default value for this setting --- -+ -The path to an X.509 certificate to use for client authentication when -communicating with Elasticsearch. - -*`var.elasticsearch.ssl.key`*:: -+ --- -* Value type is <> -* There is no default value for this setting --- -+ -The path to the certificate key for client authentication when communicating -with Elasticsearch. - -*`var.kibana.host`*:: -+ --- -* Value type is <> -* Default value is "localhost:5601" --- -+ -Sets the hostname and port of the Kibana instance to use for importing -dashboards and visualizations. For example: "myhost:5601". - -*`var.kibana.scheme`*:: -+ --- -* Value type is <> -* Default value is "http" --- -+ -Sets the protocol to use for reaching the Kibana instance. The options are: -"http" or "https". The default is "http". - -*`var.kibana.username`*:: -+ --- -* Value type is <> -* Default value is "elastic" --- -+ -The username to authenticate to a secured Kibana instance. - -*`var.kibana.password`*:: -+ --- -* Value type is <> -* Default value is "changeme" --- -+ -The password to authenticate to a secure Kibana instance. - -*`var.kibana.ssl.enabled`*:: -+ --- -* Value type is <> -* Default value is false --- -+ -Enable SSL/TLS secured communication to the Kibana instance. - -*`var.kibana.ssl.verification_mode`*:: -+ --- -* Value type is <> -* Default value is "strict" --- -+ -The hostname verification setting when communicating with Kibana. Set to -`disable` to turn off hostname verification. Disabling this has serious security -concerns. - -*`var.kibana.ssl.certificate_authority`*:: -+ --- -* Value type is <> -* There is no default value for this setting --- -+ -The path to an X.509 certificate to use to validate SSL certificates when -communicating with Kibana. - -*`var.kibana.ssl.certificate`*:: -+ --- -* Value type is <> -* There is no default value for this setting --- -+ -The path to an X.509 certificate to use for client authentication when -communicating with Kibana. - -*`var.kibana.ssl.key`*:: -+ --- -* Value type is <> -* There is no default value for this setting --- -+ -The path to the certificate key for client authentication when communicating -with Kibana.