...
| Table of Contents | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
| Info |
|---|
When using an Event Hub to send Azure Monitor or Entra ID data, use the setup instructions. |
Parameter options:
| Code Block |
|---|
{
"global_overrides": {
"debug": false
},
"inputs": {
"azure": {
"id": "<short_unique_id>",
"enabled": true,
"credentials": {
"subscription "subscription_id": "<subscription_id_value>",
"client_id": "<client_id_value>",
"client_secret": "<client_secret_value>",
"tenant_id": "<tenant_id_value>"
},
"environment": "<environment_value>",
"services": {
"vm_metrics": {
"request_period_in_seconds": "<request_period_in_seconds_value>",
"start_time_in_utc": "<start_time_in_utc_value>",
"include_resource_id_patterns": [
"<include "<include_resource_id_patterns_values>"
],
"exclude_resource_id_patterns": [
"<exclude_resource_id_patterns_values>"
]
}
}
},
"azure_event_hub": {
"id": "<short_unique_id>",
"enabled": true,
" "credentials": {
"subscription_id": "<subscription_id_value>",
"client_id": "<client_id_value>",
"client_secret": "<client_secret_value>",
"tenant_id": "<tenant_id_value>"
},
"environment": "<environment_value>",
"services": {
"event_hubs": {
"override_pull_report_frequency_seconds": "<override_pull_report_frequency_seconds_value>",
"override_consumer_client_ttl_seconds": "<override_consumer_client_ttl_seconds_value>",
"queues": {
" "queues": {
"<queue_name_value>": {
"namespace": "<namespace_value>",
"event_hub_name": "<event_hub_name_value>",
"event_hub_connection_string": "<event_hub_connection_string_value>",
"consumer_group": "<consumer_group_value>",
"blob "blob_storage_connection_string": "<blob_storage_connection_string_value>",
"blob_storage_container_name": "<blob_storage_container_name_value>",
"blob_storage_account_name": "<blob_storage_account_name_value>",
"compatibility_version": "<compatibility_version_value>",
"duplicated_messages_mechanism": "<duplicated_messages_mechanism_value>",
"override_starting_position": "<override_starting_position_value>",
" "override_tag": "<override_tag_value>",
"extend_tag": "<extend_tag_value>",
"client_thread_limit": "<client_thread_limit_value>",
"uamqp_transport": "<uamqp_transport_value>",
"partition_ids": ["<partition_id>"]
} } }, "event_hubs_auto_discover": { "resource<partition_groupid>":
"<resource_group_value>", "namespace": "<namespace_value>", "blob_storage_account_name": "<blob_storage_account_name_value>", "blob_storage_connection_string": "<blob_storage_connection_string_value>",
"consumer ]
}
}
},
"event_hubs_auto_discover": {
"resource_group": "<consumer<resource_group_value>",
"duplicated_messages_mechanism "namespace": "<duplicated_messages_mechanism<namespace_value>",
"overrideblob_pullstorage_reportaccount_frequency_secondsname": "<override<blob_pullstorage_reportaccount_frequency_secondsname_value>",
"overrideblob_consumerstorage_clientconnection_ttl_secondsstring": "<override<blob_consumerstorage_clientconnection_ttlstring_seconds_value>",
"override_starting_position "consumer_group": "<override<consumer_starting_positiongroup_value>",
"overrideduplicated_blob_storage_container_prefixmessages_mechanism": "<override<duplicated_blobmessages_storage_containermechanism_prefix_value>",
"client_thread_limit "override_pull_report_frequency_seconds": "<client_thread_limit<override_pull_report_frequency_seconds_value>",
"uamqp_transport "override_consumer_client_ttl_seconds": "<uamqp_transport<override_consumer_client_ttl_seconds_value>",
} } } }
} |
...
Parameter
...
Data type
...
Requirement
...
Value range / Format
...
Description
...
short_unique_id
...
str
...
Mandatory
...
Min length: 1, Max length: 5
...
Short, unique ID for input service, used in persistence addressing. Avoid duplicates to prevent collisions.
...
tenant_id_value
...
str
...
Mandatory
...
Min length: 1
...
Tenant ID for Azure authentication.
...
client_id_value
...
str
...
Mandatory
...
Min length: 1
...
Client ID for Azure authentication.
...
client_secret_value
...
str
...
Mandatory
...
Min length: 1
...
Client secret for Azure authentication.
...
subscription_id_value
...
str
...
Mandatory
...
Min length: 1
...
Azure subscription ID.
...
environment_value
...
str
...
Optional
...
Min length: 1
...
Differentiates environments (e.g., dev, prod). Remove if unused.
...
request_period_in_seconds_value
...
int
...
Optional
...
Min: 60
...
Custom period in seconds between data pulls, overriding default (300s).
...
start_time_in_utc_value
...
str
...
Optional
...
UTC datetime format: %Y-%m-%dT%H-%M-%SZ
...
Custom start date for data retrieval, for historical data download. Remove if unused.
...
include_resource_id_patterns_values
...
[str]
...
Optional
...
Glob patterns e.g., ["*VM-GROUP-1*"]
...
Includes resources matching patterns. Remove if unused.
...
exclude_resource_id_patterns_values
...
[str]
...
Optional
...
Glob patterns e.g., ["*VM-GROUP-1*"]
...
Excludes resources matching patterns. Remove if unused.
...
queue_name_value
...
str
...
Mandatory
...
Min length: 1
...
Name for the queue, appears in related logs.
...
event_hub_name_value
...
str
...
Mandatory
...
Min length: 1
...
Name of the Event Hub to pull events from.
...
event_hub_connection_string_value
...
str
...
Mandatory
...
Min length: 1
...
Connection string for the Event Hub.
...
consumer_group_value
...
str
...
Optional
...
Min length: 1, Default: $Default
...
Consumer group for the Event Hub. Defaults to $Default.
...
events_use_autocategory_value
...
bool
...
Optional
...
Default: true
...
Enables auto-tagging of events. This value is always true.
...
blob_storage_connection_string_value
...
str
...
Optional
...
Min length: 1
...
Connection string for blob storage, optional for Azure Blob Storage checkpointing.
...
blob_storage_container_name_value
...
str
...
Optional
...
Min length: 1
...
Blob storage container name, required if using Azure Blob Storage checkpointing.
...
blob_storage_account_name_value
...
str
...
Optional
...
Min length: 1
...
Blob storage account name, alternative to using connection string for checkpointing.
...
compatibility_version_value
...
str
...
Optional
...
Version strings
...
Compatibility version for event processing.
...
duplicated_messages_mechanism_value
...
str
...
Optional
...
One of: "local", "global", "none"
...
Deduplication mechanism for messages: local, global, or none.
...
override_starting_position_value
...
str
...
Optional
...
One of: "-1", "@latest", "[UTC datetime value]"
...
Starting position for event
fetching: from the beginning of
available data (-1), from the
latest data fetched (@fetched),
or a specific datetime (%Y-%m-
%dT%H-%M-%SZ format).
...
override_tag_value
...
str
...
Optional
...
Tag-friendly string
...
Optional tag to override the default tagging mechanism. See Event Hubs Tagging Configuration.
...
extend_tag_value
...
str
...
Optional
...
Object that can include any of the following properties: default_tag, tag_map, jmespath_refs
...
Advanced feature. Allows users to add/update various properties of the tag. If the user utilized override_tag and configured a simple tag string, this parameter will have no effect. If supplied, default_tag overrides the default tag, jmespath_refs adds/updates jmespath substitution values, and tag_map will add/update various tag paths to the pre-existing tag map. See Event Hubs Tagging Configuration.
...
override_pull_report_frequency_seconds_value
...
int
...
Optional
...
Default: 60
...
Frequency in seconds for reporting pull statistics in logs.
...
override_consumer_client_ttl_seconds_value
...
int
...
Optional
...
Default varies by service
...
Time-to-live in seconds for consumer clients, after which the collector restarts the pull cycle.
...
resource_group_value
...
str
...
Mandatory
...
Min length: 1
...
Azure resource group for event hub discovery.
...
namespace_value
...
str
...
Mandatory
...
Min length: 1
...
Namespace within Azure for event hub discovery.
...
override_blob_storage_container_prefix_value
...
str
...
Optional
...
Min length: 3, Max length: 10; Default: devo-
...
Prefix for blob storage containers created by auto-discovery service. Remove if unused.
...
uamqp_transport_value
...
bool
...
Optional
...
Default: false
...
Allows users to override/force
event hub SDK to use legacy
UAMQP transport mechanism
(true)instead of the
default/current PyAMQP
mechanism (false).
...
<partition_ids>
...
str
...
Optional
...
List of
partition
number, as["1","3","5","7"]
...
Allows to define which partitions are going to be connected by this instance of the collector. It overrides client_thread_limit_value
...
client_thread_limit_value
...
int
...
Optional
...
Min value: 1
...
Adv feature - most users should use partition_ids instead to explicitly define what partitions the collector instance will query. Number of consumer threads that the collector will create. By default, collector will create as many threads as there are consumers in the event hub.
Parameters marked as "Mandatory" are required for the collector's configuration. Optional parameters can be omitted or removed if not used, but they provide additional customization and control over the collector's behavior.
Event Hubs Tagging Configuration
Event Hubs supports multiple tagging parameters and formats to categorize and manage event data efficiently. Below are the configuration options for overriding, auto-categorizing, and extending tags.
The default configuration of the tag mapping can be found in this article.
Override tag
| Note |
|---|
Advanced setting. Please consult to Devo support before use advanced tag map. |
To customize the default tag behavior, users can configure the override_tag parameter within the Event Hub queue configuration. This parameter allows either a simple tag string or a more advanced tag mapping structure to be applied to all records.
The advanced tag map structure follows this format:
default_tag: A fallback tag applied to all records not matched by anytag_mapentry.tag_map: A list of tag entries, each containing a tag value and a JMESPath expression to match specific records.jmespath_refs: Reference variables that can be used within JMESPath expressions in thetag_map. These act as reusable values within the tag map's matching logic.
| Code Block |
|---|
override_tag:
default_tag: "tag_value"
tag_map:
- tag: "tag_value"
jmespath: "[?condition]"
- tag: "tag_value"
jmespath: "[?condition]"
...
jmespath_refs:
jmespath_ref_1: "{jmespath_expression_1}"
jmespath_ref_2: "{jmespath_expression_2}"
... |
| Code Block |
|---|
"override_tag": { " "override_starting_position": "<override_starting_position_value>", "override_blob_storage_container_prefix": "<override_blob_storage_container_prefix_value>", "client_thread_limit": "<client_thread_limit_value>", "uamqp_transport": "<uamqp_transport_value>" } } } } } |
Parameter | Data type | Requirement | Value range / Format | Description |
|---|---|---|---|---|
|
| Mandatory | Min length: 1, Max length: 5 | Short, unique ID for input service, used in persistence addressing. Avoid duplicates to prevent collisions. |
|
| Mandatory if not using an event hub connection string. | Min length: 1 | Tenant ID for Azure authentication. |
|
| Mandatory if not using an event hub connection string. | Min length: 1 | Client ID for Azure authentication. |
|
| Mandatory if not using an event hub connection string. | Min length: 1 | Client secret for Azure authentication. |
|
| Mandatory if not using an event hub connection string. | Min length: 1 | Azure subscription ID. |
|
| Optional | Min length: 1 | Differentiates environments (e.g., dev, prod). Remove if unused. |
|
| Optional | Min: 60 | Custom period in seconds between data pulls, overriding default (300s). |
|
| Optional | UTC datetime format: | Custom start date for data retrieval, for historical data download. Remove if unused. |
|
| Optional | Glob patterns e.g., | Includes resources matching patterns. Remove if unused. |
|
| Optional | Glob patterns e.g., | Excludes resources matching patterns. Remove if unused. |
|
| Mandatory | Min length: 1 | Name for the queue, appears in related logs. |
|
| Mandatory | Min length: 1 | Name of the Event Hub to pull events from. |
|
| Mandatory | Min length: 1 | Connection string for the Event Hub. |
|
| Optional | Min length: 1, Default: | Consumer group for the Event Hub. Defaults to |
|
| Optional | Default: | Enables auto-tagging of events. This value is always true. |
|
| Optional | Min length: 1 | Connection string for blob storage, optional for Azure Blob Storage checkpointing. |
|
| Optional | Min length: 1 | Blob storage container name, required if using Azure Blob Storage checkpointing. |
|
| Optional | Min length: 1 | Blob storage account name, alternative to using connection string for checkpointing. |
|
| Optional | Version strings | Compatibility version for event processing. |
|
| Optional | One of: | Deduplication mechanism for messages: local, global, or none. |
|
| Optional | One of: | Starting position for event |
|
| Optional | Tag-friendly string | Optional tag to override the default tagging mechanism. See Event Hubs Tagging Configuration. |
|
| Optional | Object that can include any of the following properties: default_tag, tag_map, jmespath_refs | Advanced feature. Allows users to add/update various properties of the tag. If the user utilized |
|
| Optional | Default: 60 | Frequency in seconds for reporting pull statistics in logs. |
|
| Optional | Default varies by service | Time-to-live in seconds for consumer clients, after which the collector restarts the pull cycle. |
|
| Mandatory for discovery | Min length: 1 | Azure resource group for event hub discovery. |
|
| Mandatory for event hubs | Min length: 1 | Namespace within Azure for event hub discovery. |
|
| Optional | Min length: 3, Max length: 10; Default: | Prefix for blob storage containers created by auto-discovery service. Remove if unused. |
|
| Optional | Default: | Allows users to override/force |
|
| Optional | List of | Allows to define which partitions are going to be connected by this instance of the collector. It overrides |
|
| Optional | Min value: 1 | Adv feature - most users should use |
Parameters marked as "Mandatory" are required for the collector's configuration. Optional parameters can be omitted or removed if not used, but they provide additional customization and control over the collector's behavior.
Event Hubs Tagging Configuration
Event Hubs supports multiple tagging parameters and formats to categorize and manage event data efficiently. Below are the configuration options for overriding, auto-categorizing, and extending tags.
The default configuration of the tag mapping can be found in this article.
Override tag
| Note |
|---|
Advanced setting. Please consult to Devo support before use advanced tag map. |
To customize the default tag behavior, users can configure the override_tag parameter within the Event Hub queue configuration. This parameter allows either a simple tag string or a more advanced tag mapping structure to be applied to all records.
The advanced tag map structure follows this format:
default_tag: A fallback tag applied to all records not matched by anytag_mapentry.tag_map: A list of tag entries, each containing a tag value and a JMESPath expression to match specific records.jmespath_refs: Reference variables that can be used within JMESPath expressions in thetag_map. These act as reusable values within the tag map's matching logic.
| Code Block |
|---|
"override_tag": {
"default_tag": "tag_value",
"tag_map": [
{
"tag": "tag_value",
"jmespath": "[?condition]"
},
{
"tag": "tag_value",
"jmespath": "[?condition]"
}
.......
],
"jmespath_refs": {
"jmespath_ref_1": "{jmespath_expression_1}",
"jmespath_ref_2": "{jmespath_expression_2}"
}
........
} |
...
The extend_tag parameter offers the following options:
default_tag: Replaces the existing default tag.jmespath_refs: Adds or updates JMESPath substitution values.tag_map: Adds or updates entries in the existing tag map. If anextend_tagentry matches an existing tag or JMESPath expression, that entry is replaced; otherwise, the new entry is appended.
Here is an example of extend_tag configuration:
| Note |
|---|
Please note that the actual internal tag structure is not displayed in this guide as it is subject to change. |
| Code Block |
|---|
extend_tag:
default_tag: "new_tag"
tag_map:
- tag: "my.app.sql"
jmespath: "[?category=='sql']"
- tag: "my.app.eh.storage"
jmespath: "[?category=='storage']"
...
jmespath_refs:
jmespath_ref_1: "{jmespath_expression_1}"
jmespath_ref_2: "{jmespath_expression_2}"
... |
| Code Block |
|---|
"extend_tag": {
"default_tag": "new_tag",
"tag_map": [
{
"tag": "my.app.sql",
"jmespath": "[?category=='sql']"
},
{
"tag": "my.app.eh.storage",
"jmespath": "[?category=='storage']"
}
........
],
"jmespath_refs": {
"jmespath_ref_1": "{jmespath_expression_1}",
"jmespath_ref_2": "{jmespath_expression_2}"
........
}
} |
If the original, internal tag structure looks like this:
| Code Block |
|---|
tag:
default_tag: "my.app.eh"
tag_map:
- tag: "my.app.eh.authentication"
jmespath: "[?category=='auth']"
- tag: "my.app.eh.sql"
jmespath: "[?category=='sql']" |
...
:
default_tag: Replaces the existing default tag.jmespath_refs: Adds or updates JMESPath substitution values.tag_map: Adds or updates entries in the existing tag map. If anextend_tagentry matches an existing tag or JMESPath expression, that entry is replaced; otherwise, the new entry is appended.
Here is an example of extend_tag configuration:
| Code Block |
|---|
"extend_tag": { "default_tag": "new_tag", "tag_map": [ { "tag": "my.app.ehsql", "tag_mapjmespath": "[?category=='sql']" }, { "tag": "my.app.eh.authenticationstorage", "jmespath": "[?category=='authstorage']" } ........ ], "jmespath_refs": { "tagjmespath_ref_1": "my.app.eh.sql{jmespath_expression_1}", "jmespath_ref_2": "[?category=='sql']{jmespath_expression_2}" }........ ]} } |
And the extend_tag configuration is applied, the resultant tag will beIf the original, internal tag structure looks like this:
| Code Block |
|---|
"tag": { default_tag: "newdefault_tag" tag_map: - tag: "my.app.eh.sql", jmespath"tag_map": "[?category=='sql']" - tag[ { "tag": "my.app.eh.storageauthentication", "jmespath": "[?category=='storageauth']" }, { - "tag": "my.app.eh.authenticationsql", "jmespath": "[?category=='authsql']" jmespath_refs: jmespath_ref_1: "{jmespath_expression_1}" jmespath_ref_2: "{jmespath_expression_2}"} ] } |
And the extend_tag configuration is applied, the resultant tag will be:
| Code Block |
|---|
"tag": {
"default_tag": "new_tag",
"tag_map": [
{
"tag": "my.app.eh.sql",
"jmespath": "[?category=='sql']"
},
{
"tag": "my.app.eh.storage",
"jmespath": "[?category=='storage']"
},
{
"tag": "my.app.eh.authentication",
"jmespath": "[?category=='auth']"
}
],
"jmespath_refs": {
"jmespath_ref_1": "{jmespath_expression_1}",
"jmespath_ref_2": "{jmespath_expression_2}"
}
} |
...
The collector should be deployed as a Docker container. Download the Docker image of the collector as a .tgz file by clicking the link in the following table:
Collector Docker image | SHA-256 hash |
|---|---|
|
Use the following command to add the Docker image to the system:
...