{ "fullnameOverride": "kfuse-agent", "nameOverride": "kfuse-agent", "datadog": { "logsEnabled": true, "logs": { "enabled": true, "containerCollectAll": true, "containerCollectUsingFiles": true, "autoMultiLineDetection": true, "dd_url": "http://kfuse-ingress-nginx-controller-internal.kfuse/ingester", "kubeStateMetricsEnabled": false, "kubeStateMetricsCore": { "enabled": true, "ignoreLegacyKSMCheck": true }, "networkMonitoring": { "enabled": false }, "orchestratorExplorer": { "enabled": true }, "apiKey": "798999171d8a0e80030b20366f2994f4", "env": [ { "name": "DD_CHECKS_TAG_CARDINALITY", "value": "orchestrator" }, { "name": "DD_CONTAINER_INCLUDE", "value": "kube_namespace:.*" }, { "name": "DD_CONTAINER_INCLUDE_LOGS", "value": "kube_namespace:.*" }, { "name": "DD_CONTAINER_EXCLUDE_METRICS", "value": "name:gclog.*" }, { "name": "DD_KUBERNETES_POD_LABELS_AS_TAGS", "value": "{\"*\":\"%%label%%\"}" }, { "name": "DD_KUBERNETES_NODE_LABELS_AS_TAGS", "value": "{\"*\":\"%%label%%\"}" }, { "name": "DD_LOGS_CONFIG_AUTO_MULTI_LINE_DEFAULT_MATCH_THRESHOLD", "value": "0.01" }, { "name": "DD_LOGS_CONFIG_AUTO_MULTI_LINE_EXTRA_PATTERNS", "value": "'^\\d{4}/\\d{2}/\\d{2}\\s\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?'^\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?'^\\d+:\\d+:\\d+(\\.\\d+)?'" } ], "apm": { "enabled": true, "portEnabled": true }, "dogstatsd": { "port": 8126, "useHostPort": false }, "processAgent": { "enabled": true }, "prometheusScrape": { "enabled": true, "version": 1 }, "additionalConfigs": [ { "configurations": [ { "send_monotonic_counter": false, "send_distribution_counts_as_monotonic": false, "send_distribution_sums_as_monotonic": false, "send_histograms_buckets": true, "max_returned_metrics": 999999, "min_collection_interval": 15 } ] } ], "autodiscovery": { "kubernetes_annotations": { "exclude": { "app": "knight", "prometheus.io/scrape": "false" }, "configurations": [ { "send_monotonic_counter": false, "send_distribution_counts_as_monotonic": false, "send_distribution_sums_as_monotonic": false, "send_histograms_buckets": true, "max_returned_metrics": 999999, "min_collection_interval": 60 } ] }, "kubernetes_container_names": [ "knight" ] }, "checksd": { "kf_openmetrics.py": "from datadog_checks.base import AgentCheck, OpenMetricsBaseCheckV2\nimport os, requests, json, time\n\ncol_names = ['metric_name', 'metric_type', 'description']\nMETRIC_TYPE_EXPANSIONS = { 'histogram' : { 'count' : {'type': 'count', 'desc_prefix' : 'Count of'}, 'sum' : {'type': 'gauge', 'desc_prefix' : 'Sum of'}, 'bucket' : {'type': 'gauge', 'desc_prefix' : 'Buckets of samples of'}, 'min' : {'type': 'gauge', 'desc_prefix' : 'Minimum value of'}, 'max' : {'type': 'gauge', 'desc_prefix' : 'Maximum value of'}, }, 'summary' : { 'count' : {'type': 'count', 'desc_prefix' : 'Count of'}, 'sum' : {'type': 'gauge', 'desc_prefix' : 'Sum of'}, 'quantile' : {'type': 'gauge', 'desc_prefix' : 'Various quantile values of'}, }}\nHELP_TEXT = '# HELP'\nTYPE_TEXT = '# TYPE'\nDEFAULT_URL = 'http://%%host%%:%%port%%/metrics'\nclass KfOpenMetricsCheck(AgentCheck):\n def __init__(self, name, init_config, instances):\n super(KfOpenMetricsCheck, self).__init__(name, init_config, instances)\n self.log.info("initializing kf_openmetrics check {}".format(instances))\n def get_default_config(self):\n return {}\n def scrape_endpoint(self, scrape_url):\n response = requests.get(scrape_url)\n if response.status_code != 200:\n raise Exception('failed to scrape endpoint: %s (error: %s)', scrape_url, response)\n return response.content.decode('utf-8')\n def prepare_mds(self, data):\n mds = {}\n content = data.splitlines()\n def _get_info(line, prefix_text):\n items = line[len(prefix_text):].split()\n return items[0], ' '.join(items[1:])\n def _add_info(metric_name, info, value):\n if metric_name not in mds:\n mds.update({metric_name:{}})\n mds[metric_name][info] = value\n def _expand_info(metric_name):\n if mds[metric_name]['type'] not in METRIC_TYPE_EXPANSIONS:\n return\n extensions = METRIC_TYPE_EXPANSIONS[mds[metric_name]['type']]\n for ext_name, extension in extensions.items():\n prefix = ""\n suffix = ""\n try:\n prefix = extension['desc_prefix']\n except KeyError:\n pass\n try:\n suffix = extension['desc_suffix']\n except KeyError:\n pass\n mds.update({'%s_%s'% (mname, ext_name): {'type' : extension['type'], 'desc' : ('%s %s %s' % (prefix, mdesc, suffix)).strip()}})\n for l in content:\n if l.startswith(HELP_TEXT):\n mname, mdesc = _get_info(l, HELP_TEXT)\n _add_info(mname, 'desc', mdesc)\n if l.startswith(TYPE_TEXT):\n mname, mtype = _get_info(l, TYPE_TEXT)\n _add_info(mname, 'type', mtype)\n _expand_info(mname)\n return mds\n def check(self, instance):\n scrape_url = ''\n try:\n scrape_url = instance.get('openmetrics_endpoint')\n if scrape_url != DEFAULT_URL:\n self.log.info("running kf_openmetrics check for instance {}".format(instance))\n data = self.scrape_endpoint(scrape_url)\n mds = self.prepare_mds(data)\n self.log.info("prepared mds len:{}, {}".format(len(mds), mds))\n self.event(\n {\n "timestamp": time.time(),\n "event_type": "OpenMetricsMetadata",\n "msg_title" : "OpenMetricsMetadata",\n "msg_text": json.dumps(mds),\n }\n )\n else:\n self.log.info("skipping kf_openmetrics check for instance {}".format(instance))\n except KeyError as ke:\n self.log.info("failed to scrape url={}, exc={}".format(scrape_url, ke))\n except requests.exceptions.InvalidURL as ie:\n self.log.info("invalid url={}, instance={}, exc={}".format(scrape_url, instance, ke))\n confd: go_expvar.yaml: |- init_config: instances: - expvar_url: http://localhost:6000/debug/vars # if you've defined \`expvar_port\` in datadog.yaml, change the port here to that value namespace: datadog.agent metrics: - path: forwarder/Transactions/RetryQueueSize - path: forwarder/Transactions/Success type: rate - path: forwarder/Transactions/Errors type: rate - path: forwarder/Transactions/ErrorsByType/DNSErrors type: rate - path: forwarder/Transactions/ErrorsByType/TLSErrors type: rate - path: forwarder/Transactions/ErrorsByType/ConnectionErrors type: rate - path: forwarder/Transactions/ErrorsByType/WroteRequestErrors type: rate - path: forwarder/Transactions/ErrorsByType/SentRequestErrors type: rate - path: forwarder/Transactions/HTTPErrors type: rate - path: dogstatsd-udp/Packets type: rate - path: dogstatsd-udp/PacketReadingErrors type: rate - path: dogstatsd-uds/Packets type: rate - path: dogstatsd-uds/PacketReadingErrors type: rate - path: dogstatsd-uds/OriginDetectionErrors type: rate - path: dogstatsd/ServiceCheckParseErrors type: rate - path: dogstatsd/ServiceCheckPackets type: rate - path: dogstatsd/EventParseErrors type: rate - path: dogstatsd/EventPackets type: rate - path: dogstatsd/MetricParseErrors type: rate - path: dogstatsd/MetricPackets type: rate - path: aggregator/Flush/ChecksMetricSampleFlushTime/LastFlush - path: aggregator/Flush/ServiceCheckFlushTime/LastFlush - path: aggregator/Flush/EventFlushTime/LastFlush - path: aggregator/Flush/MetricSketchFlushTime/LastFlush - path: aggregator/Flush/MainFlushTime/LastFlush - path: aggregator/FlushCount/ServiceChecks/LastFlush - path: aggregator/FlushCount/Series/LastFlush - path: aggregator/FlushCount/Events/LastFlush - path: aggregator/FlushCount/Sketches/LastFlush - path: aggregator/DogstatsdContexts - path: aggregator/SeriesFlushed type: rate - path: aggregator/ServiceCheckFlushed type: rate - path: aggregator/EventsFlushed type: rate - path: aggregator/NumberOfFlush type: rate - path: aggregator/DogstatsdMetricSample type: rate - path: aggregator/ChecksMetricSample type: rate - path: aggregator/ServiceCheck type: rate - path: aggregator/Event type: rate - path: aggregator/HostnameUpdate type: rate - path: scheduler/ChecksEntered - path: scheduler/Queues/.*/Size alias: scheduler.queues.size - path: scheduler/Queues/.*/Interval alias: scheduler.queues.interval - path: scheduler/QueuesCount - path: splitter/NotTooBig type: rate - path: splitter/TooBig type: rate - path: splitter/TotalLoops type: rate - path: splitter/PayloadDrops type: rate - path: logs-agent/IsRunning type: gauge - path: logs-agent/DestinationErrors type: rate - path: logs-agent/LogsDecoded type: rate - path: logs-agent/LogsProcessed type: rate - path: logs-agent/LogsSent type: rate - path: logs-agent/BytesSent type: rate - path: logs-agent/EncodedBytesSent type: rate - path: logs-agent/HttpDestinationStats/logs_0_reliable_0/idleMs type: rate alias: batch_strategy.idle_time tags: - sender:0 - path: logs-agent/HttpDestinationStats/logs_0_reliable_0/inUseMs type: rate alias: batch_strategy.in_use_time tags: - sender:0 - path: logs-agent/HttpDestinationStats/logs_1_reliable_0/idleMs type: rate alias: batch_strategy.idle_time tags: - sender:1 - path: logs-agent/HttpDestinationStats/logs_1_reliable_0/inUseMs type: rate alias: batch_strategy.in_use_time tags: - sender:1 - path: logs-agent/HttpDestinationStats/logs_2_reliable_0/idleMs type: rate alias: batch_strategy.idle_time tags: - sender:2 - path: logs-agent/HttpDestinationStats/logs_2_reliable_0/inUseMs type: rate alias: batch_strategy.in_use_time tags: - sender:2 - path: logs-agent/HttpDestinationStats/logs_3_reliable_0/idleMs type: rate alias: batch_strategy.idle_time tags: - sender:3 - path: logs-agent/HttpDestinationStats/logs_3_reliable_0/inUseMs type: rate alias: batch_strategy.in_use_time tags: - sender:3 kf_openmetrics.yaml: |- init_config: instances: - openmetrics_endpoint: http://%%host%%:%%port%%/metrics min_collection_interval: 300 clusterAgent: replicaCount: 1, enabled: true, datadog_cluster_yaml: { use_v2_api: { events: true, series: true, service_checks: true }, process_config: { process_dd_url: "http://kfuse-ingress-nginx-controller-internal.kfuse/ingester" }, events_dd_url: "http://kfuse-ingress-nginx-controller-internal.kfuse/ingester", container_collection: { enabled: false }, orchestrator_explorer: { orchestrator_dd_url: "http://kfuse-ingress-nginx-controller-internal.kfuse" }, manifest_collection: { enabled: false }, admissionController: { enabled: false } }, agents: { image: { tagSuffix: "jmx", useConfigMap: true }, customAgentConfig: { skip_ssl_validation: false, enable_stream_payload_serialization: false, use_v2_api: { events: true, series: true, service_checks: true, process_config: { process_dd_url: "http://kfuse-ingress-nginx-controller-internal.kfuse/ingester" }, events_dd_url: "http://kfuse-ingress-nginx-controller-internal.kfuse/ingester" }, container_collection: { enabled: false }, orchestrator_explorer: { orchestrator_dd_url: "http://kfuse-ingress-nginx-controller-internal.kfuse" }, manifest_collection: { enabled: false }, logs_config: { logs_dd_url: "kfuse-ingress-nginx-controller-internal.kfuse:80", logs_no_ssl: true, use_http: true, auto_multi_line_detection: true, use_v2_api: false, apm_config: { enabled: true, apm_non_local_traffic: true, apm_dd_url: "kfuse-ingress-nginx-controller-internal.kfuse:80/ingester" } }, metadata_providers: [ { "name": "host", "interval": 300 } ] } } }