#type=humio | #kind=metrics | m1>0 | name=/storage-(read|write)/
| timechart(name, function={groupby(#vhost, function=max(m1)) | sum(_max)}, span=1m)

#kind=logs | #type=humio
| class=/BucketStorageDownloadExecutorSingle/
| /download[a-z ]+completed.+humiodata/
| case { reason = /IQ-(.+)/ | aggReason := "Query";
         * | aggReason := reason }
| timechart(aggReason, function=[sum(size)], limit=50)

#type = humio | #kind=metrics | name="bucket-storage-download-requests-cap-size-hits"
| timechart(function=max(value), minSpan=1m)

#type=humio #kind=logs class="*QuerySessions*" "creating new query"
| join({#kind=logs | #type=humio | not "segment-modified" | class=/BucketStorageDownloadExecutorSingle/ /download[a-z ]+completed.+humiodata\.(?<segment>[^\s]+)/
    |   groupby(reason, function=[count(segment, distinct=true, as=segments), sum(size, as=size)])
       }, key=reason, field=queryID, include=[segments, size])
| sizeInGB:=size/(1024*1024*1024)
| table([dataspace, queryID, start, end, segments, sizeInGB], sortby=sizeInGB, limit=100)

#type = humio | #kind=metrics | name="bucket-storage-download-queue-free-slots"
| timechart(function=min(value), minSpan=1m)

#type = humio | #kind=metrics
| name = "bucket-storage-*"
| groupby([cluster,name],function=[min(value),max(value)]) | name=~replace("^bucket-storage-", with="")
| sort([cluster,name],order=ascending)

#type = humio | #kind=metrics | name="bucket-storage-upload-latency-max"
| timechart(function=max(value), minSpan=1m) | _max := _max / 1000

#type = humio | #kind=metrics | name="bucket-storage-segment-downloads-in-progress"
| timechart(function=max(value), minSpan=1m)

#kind = "logs" | @rawstring = "*Mirrored from*" | to=* AND hostId=* | timeChart( function=[sum("fileSizeBytes", as="All Nodes")], span=12h)

#kind = "logs" | @rawstring = "*Mirrored from*" | to=* | timeChart(function=[sum("fileSizeBytes", as="All Nodes")], span=12h)

#kind = "logs" | @rawstring = "*Mirrored from bucket*" | to=* | timeChart(function=[sum("fileSizeBytes", as="All Nodes")], span=12h)

#kind = "logs" | @rawstring = "*Mirrored from bucket*" | to=* | timeChart(series=to, function=[sum("fileSizeBytes")], span=12h) | to := format(format="Node %s", field=[to])

#kind = "logs" | @rawstring = "*Mirrored from*" | to=* AND hostId =* | bucketConnection := format(format="From %s to %s", field=[hostId, to]) | timeChart(series=bucketConnection, function=[sum("fileSizeBytes")], span=12h)

$humio/insights:ingestLatencyByHost()

#type=humio #kind=logs | loglevel=ERROR | regex("(?<exception>[\w\.]+Exception)", strict=false) | /ERROR[^-]+- (?<msg>[^:=]+)/ | desc:=format("%s | %s | %s",field=[class, msg, exception]) | groupby([desc], function=[count(), collect(#vhost, separator=" "), max(@timestamp)]) | ago_sec:=(now()-_max)/1000 | round(ago_sec) | replace("^(.{15}).*", with="$1\.\.\.", field=#vhost, as="hosts") | table([_count, ago_sec, hosts, desc], sortby=_count)

$humio/insights:slowWarningsInQueries()

#kind!=* @rawstring="*fatal error*" _CMDLINE=*

#type=humio #kind=activity category=AggregateAlert message != "Problem invoking action. If all actions fail, they will be retried" | severity=~in(values=["Warning", "Error"]) | format(format="%s/%s", field=[dataspace, alertName], as=Alert) | timechart(Alert, limit=100)

#type=humio #kind=requests| status>=400 status<500 | timechart(status)

$humio/insights:searchQueue()

$humio/insights:globalSnapshotSize()

#type=humio #kind=activity category=ScheduledSearch severity="Warning" message="Problem firing action" | format(format="%s/%s", field=[dataspace, scheduledSearchName], as=ScheduledSearch) | timechart(ScheduledSearch, limit=100)

#type=humio #kind=metrics name=missing-cluster-nodes | timechart(function=max(value))

#type=humio #kind=activity category=AggregateAlert message="Problem invoking action. If all actions fail, they will be retried" | format(format="%s/%s", field=[dataspace, alertName], as=Alert) | timechart(Alert, limit=100)

#type=humio #kind=logs loglevel=ERROR
| regex("(?<exception>[\w\.]+Exception)", strict=false)
| /ERROR[^-]+- (?<msg>[^:=]+)/
| desc:=format("%s | %s | %s",field=[class, msg, exception])
| timechart(desc, limit=50)

#type=humio #kind=activity category=FilterAlert severity="Warning" message="Problem invoking action. If all actions fail, they will be retried" | format(format="%s/%s", field=[dataspace, alertName], as=Alert) | timechart(Alert, limit=100)

#type=humio #kind=metrics name=failed-http-checks
| timechart(#vhost, function=max(m1), limit=100, minSpan=1m)

#type=humio #kind=activity category=Alert message != "Problem invoking action" | severity=~in(values=["Warning", "Error"]) | format(format="%s/%s", field=[dataspace, alertName], as=Alert) | timechart(Alert, limit=100)

$humio/insights:globalTransactions()

#type=humio #kind=activity category=Fdr | severity=~in(values=["Warning", "Error"]) | format(format="%s/%s", field=[dataspace, fdrFeedName], as=FdrFeed) | timechart(FdrFeed, limit=100)

#type=humio #kind=logs class=/c\.h\.c\.ClusterManagement/ "missing segment" | timechart(#vhost, limit=100)

#type=humio #kind=logs class=/c\.h\.c\.ShutdownHook/ "server closed" | timechart(#vhost)

#type=humio #kind=requests | status>=500 | timechart(status)

#type=humio #kind=activity category=Alert severity="Warning" message="Problem invoking action" | format(format="%s/%s", field=[dataspace, alertName], as=Alert) | timechart(Alert, limit=100)

#type=humio #kind=activity category=ScheduledSearch message != "Problem firing action" | severity=~in(values=["Warning", "Error"]) | format(format="%s/%s", field=[dataspace, scheduledSearchName], as=ScheduledSearch) | timechart(ScheduledSearch, limit=100)

#type=humio #kind=activity category=FilterAlert message != "Problem invoking action. If all actions fail, they will be retried" | severity=~in(values=["Warning", "Error"]) | format(format="%s/%s", field=[dataspace, alertName], as=Alert) | timechart(Alert, limit=100)

$humio/insights:cpuUsagePercent()

$humio/insights:ingestErrors()

#type=humio ((#kind=nonsensitive name="cluster_versions") OR (#kind=logs "Http reachability" me=*))
| case {
  test(#vhost == host) | self_reported := "true";
  name="cluster_versions" | peer_reported := "true";
  "Http reachability" me=* | host := hostId | host_unreachable := "true"
}
| bucket(field=host, function=[
    {peer_reported = true | count(as=peer_reported_count)},
    {self_reported = true | count(as=self_reported_count)},
    {host_unreachable = true | count(as=host_unreachable_count)}
  ], limit=500, minSpan=5m)
| self_reported_count=0 peer_reported_count>0 host_unreachable_count=0
| @timestamp := _bucket | timechart(host, limit=100, minSpan=5m)

#type=humio #kind=nonsensitive name=meminfo
| used:=100*Cached/MemTotal
| timechart(#vhost, function=max(used), limit=100)

#type=humio #kind=nonsensitive name="stat_cpu"
| percentCPU:=(user + system + nice) /(user + system + nice + idle) * 100
| groupBy([#type], function=[percentile(percentCPU, percentiles=[99])])
| select([_99])

#type=humio #kind=metrics name="event-latency"
| timechart(#vhost, function=[max(max, as=max),avg(median, as=median)], limit=100, minSpan=1m)
| median:=median/1000 | max:=max/1000

#kind=metrics | name="event-latency"
| groupBy([#kind], function=[percentile("max", percentiles=[90])])
| p90 := _90/1000
| select([p90])

#type=humio #kind=nonsensitive name="stat_cpu"
| percentCPU:=(user + system + nice) /(user + system + nice + idle) * 100
| timechart(#vhost, function=avg(percentCPU), limit=100, minSpan=1m)

#type=humio #kind=nonsensitive name="stat_cpu"
| percentCPU:=(user + system + nice) /(user + system + nice + idle) * 100
| groupBy([#type], function=[percentile(percentCPU, percentiles=[90])])
| select([_90])

#kind=metrics
| name="event-latency"
| timechart(limit=20, function=percentile("max", percentiles=[50,75,90,95,99,100]), minSpan=1m, buckets=1500)

#type=humio  #kind=nonsensitive name=meminfo | used:=100*Cached/MemTotal
| groupBy([#type], function=[avg(field=used, as=avg_mem)])
| select([avg_mem])

#kind=metrics | name="event-latency"
| groupBy([#kind], function=[percentile("max", percentiles=[99])])
| p99 := _99/1000
| select([p99])

#type=humio  #kind=nonsensitive name=meminfo | used:=100*Cached/MemTotal
| groupBy([#type], function=[max(field=used, as=max_mem)])
| select([max_mem])

#type=humio (#kind=nonsensitive or #kind=metrics) (name="stat_cpu" or name=meminfo or name="event-latency") | used:=100*Cached/MemTotal
| percentCPU:=(user + system + nice) /(user + system + nice + idle) * 100
| groupBy([#type], function=[percentile(percentCPU, percentiles=[99, 90], as="cpu"), avg(field=used, as=avg_mem), max(used, as=max_mem), percentile("max", percentiles=[99, 90], as="latency")])
| case {
    (cpu_99 >90 cpu_90 > 80) or (avg_mem > 80 max_mem > 95) or (latency_99 > 60000) | scaleStatus := "Scale up";
    cpu_99 <30 cpu_90 < 20 avg_mem < 20 max_mem < 50 | scaleStatus := "Consider scaling down";
    * | scaleStatus := "Scaling is optimal";
}
| select([scaleStatus])

#type=humio #kind=logs
| " INFO  c.h.q.QuerySessionsImpl "
| "static query ended:" or "static part of live query ended:"
| time > 100
| time := time / 1000
| timechart(#vhost, function=percentile("time", percentiles=[75]), limit=500, span=1hr)

$humio/insights:jvmHiccup()

#type="humio" #kind="logs" @rawstring="*Http reachability check failed polling from*"
| timeChart(hostId, function={count(field=hostId, distinct=true)}, limit=50, minSpan=1m)

#type=humio #kind=nonsensitive name="cluster_versions" | timechart(version, function=count(host, distinct=true), limit=100, minSpan=1m)  // All nodes report version of all other nodes.

#type=humio #kind=nonsensitive name=processstatus | timechart(#vhost, function=max(VmRSS), limit=50, minSpan=1m)

#type=humio #kind=metrics name=jvm-heap-usage-percent | timechart(#vhost, function=max(value, as=percent), limit=50, minSpan=1m)

#type=humio | timechart(#vhost, limit=100, unit="1/s")

#type=humio #kind=metrics name=secondary-disk-usage  | timechart(#vhost, function=max(value), limit=100, minSpan=1m)

#type=humio  #kind=nonsensitive name=fdcount | timechart(#vhost, function=max(fdcount), limit=50, minSpan=1m)

// Chart mini segment disk usage as percentage of available space on local disk: (Humio v1.44+)
#type=humio #kind=logs class=/DiskSpaceJob/
| "Scanning localdatasource found these byte size totals locally"  OR "DiskSpaceJob: Status at start"
| timechart(#vhost, function=[max(miniBytes,as=minis),max(desiredUsedBytes,as=maxUse)], limit=50, minSpan=5m)
| minis:=100*minis/maxUse | drop(maxUse)

$humio/insights:cpuLoad()

#type=humio  #kind=nonsensitive name=meminfo | used:=100*Cached/MemTotal | timechart(#vhost, function=max(used), limit=50, minSpan=1m)

#type=humio #kind=metrics name="direct-memory-allocated" | timechart(#vhost, function=max(value),limit=100, minSpan=1m)

#type=humio #kind=metrics name=jvm-heap-usage | timechart(#vhost, function=max(value, as=bytes), limit=50, minSpan=1m)

#type=humio #kind=logs class=/c\.h\.c\.ShutdownHook/ "server closed" | timechart(#vhost, limit=100)

#type=humio #kind=nonsensitive name="cluster_versions"
| groupby(version, function=count(host, distinct=true, as=nodes))
| table([nodes, version]) // All nodes report version of all other nodes.

#type=humio #kind=nonsensitive name=lscpu |
groupby(#vhost, function=selectLast([cpus, onlineCpus, modelName, numaNodes, threadsPerCore, virtualizationType, sockent, hypervisorVendor, l1dCache, l1iCache, l2Cache, l3Cache]))
| sort(#host)

#type=humio #kind=metrics name=primary-disk-usage  | timechart(#vhost, function=max(value), limit=100, minSpan=1m)

#type=humio #kind=logs class=/\.JVMStats/ "GC stats" | timechart(#vhost, function={groupby(name, function=counterAsRate(time_ms)) | sum(_rate)}, limit=100, minSpan=1m)

#type=humio #kind=metrics | name = /thread-pool-queue-size\/(?<pool>[\s\S]*)/
| timeChart(pool, function=max(value), limit=50)

#type=humio  #kind=nonsensitive name=netdev | timechart(#vhost, function=
  {groupby(device,
    function={[
      min(rx_bytes, as=rx_min), max(rx_bytes, as=rx_max),
      min(tx_bytes, as=tx_min), max(tx_bytes, as=tx_max)]
    | rx:=rx_max - rx_min | tx:=tx_max - tx_min})
| [sum(rx, as=rx), sum(tx, as=tx)]}
, unit="1/s", limit=100, minSpan=1m)

#type=humio #kind=nonsensitive name="stat_tasks" | time:=(utime+stime) | timechart(threadGroup, function=avg(time, as=ticks), limit=100, minSpan=1m)

#type=humio #kind=metrics name="temp-disk-usage-bytes" | timechart(series=#vhost, function=max("value"), limit=50, minSpan=1m)

#type=humio #kind=metrics name=missing-cluster-nodes | timechart(function=max(value), limit=50, minSpan=1m)

$humio/insights:cpuUsagePercent()

#type=humio #kind=logs "Http reachability" me=*
| groupby([me, hostId])
| source:=rename(me) | target:=rename(hostId) | weight:=rename(_count)

#type=humio #kind=metrics name=cluster-time-skew | timechart(#vhost, function=max(value), limit=100, minSpan=1m)

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/ or (uri=/_bulk\/?$/ and uri!=/api/))
|uri match{
  /collector/ => kind:= "hec";
  /hec/ => kind:= "hec";
  /elastic-bulk/ => kind:="elastic_bulk";
  /_bulk\/?$/ => kind:="elastic_bulk";
  /logplex/ => kind:="logplex";
  /humio-structured/ => kind:="humio-structured";
  /humio-unstructured/ => kind:="humio-unstructured";
  /raw/ => kind:="raw";
  * => kind:=uri
}
| timechart(kind, unit="1/s")

$humio/insights:datasourcesHittingMaxAutoshards()

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/ or (uri=/_bulk\/?$/ and uri!=/api/)) status>=400 | timechart(status, unit="1/s")

#type=humio #kind=metrics name=livequery-count
| timechart(#vhost, function=max(value), limit=100, minSpan=1m)

$humio/insights:datasourcesAutoShardingTable()

#type=humio #kind=metrics name=/^ingest-bytes(\/[^\/]+)?\/(?<repo>.*)$/
| timechart(repo,
       function={ groupby([#vhost, name], function=avg(m5)) | sum(_avg)},
       limit=100, minSpan=1m, unit="bytes/sec to bytes/day")

#type=humio #kind=logs class=c.h.c.c.IngestPartitionCoordinator "Wrote changes to current digest partitions" | changedAndAddedPartitions := changedPartitions + addedPartitions | timechart(#vhost, function=sum(changedAndAddedPartitions), limit=100, minSpan=1m)

#type=humio #kind=metrics name=/^ingest-parsing(\/[^\/]+)?\/(?<parser>[^\/]+\/[^\/]+)$/
| groupby([parser, name, #vhost], function=[avg(mean, as=timePerEvent), avg(m1, as=eventsPerSecond), {time:=m1 * mean| avg(time, as=time)}])
| groupby(parser, function=[sum(time, as=timeInParserPerSecond), avg(timePerEvent, as=timePerEvent), sum(eventsPerSecond, as=eventsPerSecond)])
| timeInParser:= format("%d", field=timeInParserPerSecond)
| timePerEvent:= format("%,.2f", field=timePerEvent)
| eventsPerSecond:= format("%,.2f", field=eventsPerSecond)
| table([parser, timeInParser, eventsPerSecond, timePerEvent], sortby=timeInParserPerSecond, limit=20)

$humio/insights:ingestPerHost()

$humio/insights:IngestPerHostEventsPerSecond()

#type=humio #kind=metrics (name=time-livequery or name = "time-digest")
| timechart(name, function={groupby(#vhost, function=max(value)) | sum(_max)}, limit=100, minSpan=1m)

$humio/insights:ingestErrors()

#type=humio #kind=metrics | /name=ingest-offset-lowest\/(?<partition>\d+)/  | timechart(partition,function=counterasrate(value), limit=50, span=1h)

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/ or (uri=/_bulk\/?$/ and uri!=/api/)) | timechart(unit="1/s")

$humio/insights:ingestLatencyByHost()

#type=humio #kind=metrics name=kafka-request-events | timechart(#vhost, function=[counterAsRate(count, as=c), avg(mean, as=m)], limit=100, minSpan=1m) | "Events Per Second":=c*m | drop([c,m])

#type=humio #kind=metrics | name=/event-latency-partition\/(?<p>\d+)/  | timechart(p,function=[avg(max,as=max),avg(mean,as=mean)], limit=50, minSpan=1m) | max:=max/1000 | mean:=mean/1000

$humio/insights:cpuUsagePercent()

#type=humio #kind=metrics name=/^ingest-parsing(\/[^\/]+)?\/[^\/]+\/[^\/]+$/
| time:= mean * m1
| timechart(function={groupby([#vhost, name], function=avg(time)) | sum(_avg)}, limit=100, minSpan=1m)

$humio/insights:datasourcesChangingAutoShards()

#type=humio #kind=metrics | name=/^written-events(\/[^\/]+)?\/(?<repo>.*)$/
| timechart(repo, function={groupby([#vhost, name], function=avg(m5)) | sum(_avg)}, limit=100, minSpan=1m)

#type=humio class=/c\.h\.c\.ShutdownHook/ "server closed" | timechart(#vhost, limit=100)

$humio/insights:datasourcesPerRepo()

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/ or (uri=/_bulk\/?$/ and uri!=/api/))
| timechart(function=percentile(time))

#type=humio class=c.h.q.LocalLivequeryMonitor "Ingest delay exceeds threshold, canceling livequeries." | timechart(#vhost, limit=100, minSpan=1m)

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/ or (uri=/_bulk\/?$/ and uri!=/api/))
| timechart(function=percentile(contentLength))

#type=humio #kind=logs class=c.h.j.KafkaStatusLoggerJob
| topic_num_out_of_sync_replicas > 0
| timechart(topic_name, function=count(topic_partition, distinct=true))

#type=humio #kind=logs class = c.h.j.KafkaStatusLoggerJob
| topic_num_out_of_sync_replicas > 0
| topic_name="*humio-ingest"
| groupby([topic_name,topic_partition])
| count()

#type=humio #kind=metrics name=ingest-writer-uncompressed-bytes | timechart(#vhost, function=max(m1), limit=100, minSpan=1m)

#type=humio #kind=logs class=c.h.j.KafkaStatusLoggerJob
| groupby([topic_name, topic_partition], function=[collect(topic_leader, separator=" "), collect(topic_replicas, separator=" "), collect(topic_in_sync_replicas, separator=" "), collect(topic_is_in_sync, separator=" ")],limit=200)
| table([topic_name, topic_partition, topic_is_in_sync, topic_leader, topic_replicas, topic_in_sync_replicas], sortby=[topic_is_in_sync, topic_name, topic_partition], order=ascending)

#type=humio #kind=metrics name=kafka-request-bytes | timechart(#vhost, function=max(p75), limit=50, minSpan=1m)

#type=humio #kind=metrics name=global-publish-wait-for-value | timechart(#vhost, function=max(p75), limit=100, minSpan=1m)

#type=humio #kind=logs class=c.h.j.KafkaStatusLoggerJob
| topic_num_out_of_sync_replicas > 0
| topic_name="*transientChatter-events"
| groupby([topic_name,topic_partition])
| count()

#type=humio #kind=metrics name=/kafka-chatter-by-kind-bytes\/(?<kind>[^,]+)/ | timechart(kind, function={groupby(#vhost, function=avg(m1)) | sum(_avg)}, limit=100, minSpan=1m)

#type=humio #kind=metrics name=global-publish-wait-for-value | timechart(#vhost, function=max(m1), limit=100, minSpan=1m)

#type=humio #kind=logs class = c.h.j.KafkaStatusLoggerJob
| topic_num_out_of_sync_replicas > 0
| topic_name="*global-events"
| groupby([topic_name,topic_partition])
| count()

#type=humio #kind=metrics name=kafka-ingestqueue-put | timechart(#vhost, function=max(m1), limit=100, minSpan=1m, unit="1/s")

#type=humio #kind=metrics name=kafka-chatter-bytes | timechart(#vhost, function=avg(m1), limit=100, minSpan=1m)

#type=humio #kind=metrics name=kafka-chatter-put | timechart(#vhost, function=avg(m1), limit=100, minSpan=1m)

$humio/insights:globalTransactions()

#type=humio "kafka-metrics ingest-consumer" | class=*IngestQueue* | timechart(partition, function=max("records-lag-max"), limit=100)

#type=humio #kind=metrics name=kafka-ingestqueue-put | timechart(#vhost, function=max(p75), limit=50, minSpan=1m)

#type=humio #kind=metrics | name=/^lars-resolve-assets-calls\/(?<asset_type>.*)/
| groupBy(["asset_type", #vhost], function=[min(count, as=min_count), max(count, as=max_count)])
| change := max_count - min_count
| groupBy([asset_type], function=sum(change))

#type=humio #kind=logs | loglevel="WARN" | "Failed to fetch assets from LARS for uri" | count()

#type=humio #kind=metrics | name=/^lars-resolve-assets-request-duration/
| timechart(#vhost, function=[max(max, as=max),avg(median, as=median)], limit=100, minSpan=1m)

#type=humio #kind=metrics | name=/^lars-assets-cache-calls-to-(?<operation>.*)/
 | timeChart(operation, function={groupBy([#vhost], function=counterAsRate("value")) | sum("_rate")}, minSpan=2m, limit=2)

#type=humio #kind=metrics | name="lars-assets-cache-values-size" | timeChart(#vhost, function=avg(value), limit=10, minSpan=1m)

#type=humio #kind=metrics | name="lars-assets-cache-keys-size" | timeChart(#vhost, function=avg(value), limit=10, minSpan=1m)

#type=humio #kind=metrics | name=/^lars-assets-cache-calls-to-(?<operation>.*)/
| [
    {operation = "get" | gets := sum(field=value)},
    {operation = "load" | loads :=  sum(field=value)}
]
| cache_hit_ratio := 1 - (loads / gets) | select([cache_hit_ratio])

#type=humio #kind=metrics | name=/^lars-resolve-assets-calls\/(?<asset_type>.*)/
| timechart(asset_type, function={groupby(#vhost, function=avg(m1)) | sum(_avg)}, limit=16, minSpan=1m)

#kind="logs" | "File was loaded. fileName" | timechart(function=max("sizeOnDisk"), minSpan=1m)

#kind="metrics" | name="lookup-tables-number-of-lookup-tables" | timechart(function=max("value"), minSpan=1m)

#kind="metrics" | name="lookup-tables-number-of-files" | timechart(function=max("value"), minSpan=1m)

#kind="metrics" | name="tables-loading-parser-tasks" | timeChart(function=max("value"), minSpan=1m)

#kind="logs" | "File was loaded. fileName" | timechart(function=max("timeMs"), minSpan=1m)

#kind="metrics" | name="lookup-tables-number-of-index-structures" | timechart(function=max("value"), minSpan=1m)

#kind="metrics" | name="lookup-tables-size-of-cached-files" | timechart(function=max("value"), minSpan=1m)

#kind="logs" | "File was loaded. fileName" | timechart(function=max("parsedSize"), minSpan=1m)

#kind="metrics" | name="lookup-tables-loading-tasks" | timeChart(function=max("value"), minSpan=1m)

#kind="metrics" | name="lookup-tables-number-of-queries" | timechart(function=max("value"), minSpan=1m)

$humio/insights:ingestPerHost()

$humio/insights:IngestPerHostEventsPerSecond()

$humio/insights:searchQueue()

#type=humio #kind=logs | loglevel=ERROR | regex("(?<exception>[\w\.]+Exception)", strict=false) | /ERROR[^-]+- (?<msg>[^:=]+)/ | desc:=format("%s | %s | %s",field=[class, msg, exception]) | groupby([desc], function=[count(), collect(#vhost, separator=" "), max(@timestamp)]) | ago_sec:=(now()-_max)/1000 | round(ago_sec) | table([_count, ago_sec, desc, #vhost], sortby=_count)

#type=humio #kind=requests | timechart(#vhost, unit="1/s")

$humio/insights:ingestLatencyByHost()

$humio/insights:cpuUsagePercent()

$humio/insights:ingestErrors()

#type=humio #kind=logs loglevel=ERROR
| regex("(?<exception>[\w\.]+Exception)", strict=false)
| /ERROR[^-]+- (?<msg>[^:=]+)/
| desc:=format("%s | %s | %s",field=[class, msg, exception])
| timechart(desc, limit=50)

$humio/insights:jvmHiccup()

#type=humio #kind=metrics name=/^ingest-parsing(\/[^\/]+)?\/(?<parser>[^\/]+\/[^\/]+)$/
| groupby([parser, name, #vhost], function=[avg(mean, as=timePerEvent), avg(m1, as=eventsPerSecond), {time:=m1 * mean| avg(time, as=time)}])
| groupby(parser, function=[sum(time, as=timeInParserPerSecond), avg(timePerEvent, as=timePerEvent), sum(eventsPerSecond, as=eventsPerSecond)])
| timeInParser:= format("%d", field=timeInParserPerSecond)
| timePerEvent:= format("%,.2f", field=timePerEvent)
| eventsPerSecond:= format("%,.2f", field=eventsPerSecond)
| table([parser, timeInParser, eventsPerSecond, timePerEvent], sortby=timeInParserPerSecond, limit=20)

#kind=logs class=/ParserLimitingJob/ /Top element for parser id=((?<parserID>[^/]+)\/)?(?<repo>[^/]+)\/(?<parserName>\S+)/ //id=/humio\/humio/i
| ratio:=10000*cost/threshold | round(ratio) | ratio:=ratio/100
| groupby([repo, parserName], function=[sum(cost, as=costSum), count(as=count), max(ratio, as=ratio)]) | rename(ratio, as=highestRatio)
| sort(costSum)

$humio/insights:ingestPerHost()

#kind=logs class=/ParserLimitingJob/ /Top element for parser id=((?<parserID>[^/]+)\/)?(?<repo>[^/]+)\/(?<parserName>\S+)/
| pct:=100*costSum/threshold
| timechart(function=max(pct), minSpan=10s, buckets=500)

#type=humio #kind=logs statuscode=503
/msg=(?<msg>Ingest parsing exceeded the acceptable amount of time[^\.]+)\. exception/
| timechart(buckets=500)

$humio/insights:ingestLatencyByHost()

$humio/insights:cpuUsagePercent()

#type=humio #kind=metrics name=/^ingest-parsing(\/[^\/]+)?\/[^\/]+\/[^\/]+$/
| time:= mean * m1
| timechart(function={groupby([#vhost, name], function=avg(time)) | sum(_avg)}, limit=100, minSpan=1m)

#type=humio class=/c\.h\.c\.ShutdownHook/ "server closed" | timechart(#vhost, limit=100)

#type=humio #kind=logs | loglevel=WARN | class = c.h.d.ParserLimitingJob
| "Setting reject ingest for"
| groupby([id], function=[selectLast(ratio), count()]) | sort(_count)

#type=humio #kind=activity category = PermissionAssignment subCategory = groupAssignments
| wildcard(field=orgId, includeEverythingOnAsterisk=true, pattern=?orgId)
| groupBy(groupId, function=selectLast([group, assignments, assetAssignments]))
| sort(assignments, limit=10)

#type=humio #kind=metrics name=/^create-requester-time\/(?<orgId>.*)$/
| orgId=?orgId
| timechart(orgId, function={groupby(#vhost, function=avg(mean)) | sum(_avg)}, limit=16, minSpan=1m)

#type=humio #kind=metrics name=/^create-requester-time\/(?<orgId>.*)$/
| orgId=?orgId
| time := m1*mean
| timechart(orgId, function={groupby(#vhost, function=avg(time)) | sum(_avg)}, limit=16, minSpan=1m)

#type=humio #kind=activity category = PermissionAssignment subCategory = userAssignments
| wildcard(field=orgId, includeEverythingOnAsterisk=true, pattern=?orgId)
| groupBy(userId, function=selectLast([assignments, userAssignments, assetAssignments, groupAssignments]))
| sort(assignments, limit=10)

#type=humio #kind=metrics name=/^create-requester-time\/(?<orgId>.*)$/
| orgId=?orgId
| timechart(orgId, function=avg("m1"), limit=16, minSpan=1m)

#type=humio #kind=metrics name=/^check-permission\/(?<permission>.*)$/
| timechart(permission, function={groupby(#vhost, function=avg(m1)) | sum(_avg)}, limit=16, minSpan=1m)

#type=humio #kind=metrics name = /^user-permissions-(?<name2>.*)/
| timeChart(name2, function={groupBy([#vhost], function=counterAsRate("count")) | sum("_rate")}, minSpan=1m, limit=2)

#type=humio #kind=activity category = PermissionAssignment subCategory = numberOfGroups
| wildcard(field=orgId, includeEverythingOnAsterisk=true, pattern=?orgId)
| table([orgId, numberOfGroups], sortby=numberOfGroups, limit=10)

#type=humio #kind=logs
| @rawstring=/Segment rebalance by host/
| reason="EvictedHostRemoval" or reason="EvictedHostReplacement" // Both cases end up removing a host from owners
| timeChart(removeVhost, function=[sum(bytes, as="Bytes")], limit=100)

#type=humio #kind=logs
| @rawstring=/Segment rebalance by host/
| reason="DecreaseReplication"
| timeChart(removeVhost, function=[sum(bytes, as="Bytes")], limit=100)

#type=humio #kind=logs
| @rawstring=/Segment rebalance by host/
| reason="EvictedHostReplacement"
| timeChart(addVhost, function=[sum(bytes, as="Bytes")], limit=100)

#type=humio #kind=logs
| @rawstring=/Segment rebalance by host/
| reason="ImproveZoning"
| timeChart(addVhost, function=[sum(bytes, as="Bytes")], limit=100)

#type=humio #kind=logs
| @rawstring=/Segment rebalance by host/
| reason="IncreaseReplication"
| timeChart(addVhost, function=[sum(bytes, as="Bytes")], limit=100)

#type=humio #kind=logs
| @rawstring=/Segment rebalance by host/
| reason="ImproveDiskBalance"
| timeChart(addVhost, function=[sum(bytes, as="Bytes")], limit=100)

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/ or (uri=/_bulk\/?$/ and uri!=/api/))
|uri match{
  /raw/ => kind:="raw";
  /collector/ => kind:= "hec";
  /hec/ => kind:= "hec";
  /elastic-bulk/ => kind:="elastic_bulk";
  /_bulk\/?$/ => kind:="elastic_bulk";
  /logplex/ => kind:="logplex";
  /humio-structured/ => kind:="humio-structured";
  /humio-unstructured/ => kind:="humio-unstructured";
  * => kind:=uri
}
| timechart(kind, unit="1/s")

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/) | timechart(unit="1/s")

#type=humio #kind=requests | uri=/\/query$/ internal=false | timechart(#vhost, limit=100, unit="1/m")

#type=humio #kind=requests | uri=/\/query|queryjob/ internal=false | timechart(#vhost, limit=100, unit="1/s")

#type=humio #kind=requests | uri=/\/query|queryjob/ internal=true | timechart(#vhost, limit=100, unit="1/s")

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/)
| timechart(function=percentile(time))

#type=humio #kind=requests | uri=/\/query\/?$|queryjobs\/?$/ method=POST internal=false | timechart(#vhost, limit=100, unit="1/m")

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/)
| timechart(function=percentile(contentLength))

#type=humio #kind=requests status>=400 | timechart(status, unit="1/s")

#type=humio #kind=requests (uri=/api\/v1\/ingest\// or uri=/services\/collector/) status>=400 | timechart(status, unit="1/s")

#type=humio #kind=requests | uri=/\/(?<repo>[^\/]+)\/queryjobs$/ method=POST internal=false | timechart(repo, limit=100, unit="1/m")

#type=humio #kind=requests
| timechart(function=percentile(time, percentiles=[50, 90, 99, 100]))

#type=humio #kind=requests | timechart(unit="1/s")

#type=humio #kind=logs class=/c\.h\.q\.m\.MapReduceQueryExecutor/ | "Starved searching" | timechart(#vhost, limit=100, minSpan=1m)

#type=humio #kind=logs class=/c\.h\.s\.e\.InternalClusterQueryJobEndpoint/ "serializing large state" | timechart(function=max(size))

#type=humio #kind=requests | uri=/\/query|queryjob/ internal=false | timechart(#vhost, limit=100, minSpan=1m, unit="1/s")

#type=humio #kind=metrics name=queries
| timechart(#vhost, function=counterasrate(count), limit=100, minSpan=1m, unit="1/s to 1/m")

$humio/insights:cpuUsagePercent()

#type=humio #kind=metrics name=query-delta-total-memory-allocation | timechart(function={groupby(#vhost, function=max(mean)) | sum(_max, as=mean)})

#type=humio #kind=logs class=/c\.h\.s\.e\.InternalClusterQueryJobEndpoint/ "serializing large state"
| queryID =~ join({#type=humio #kind=logs class="*QuerySessions*" "creating new query"}, mode=left, include=query)
| top([dataspace, queryID, query], max=size)
| _max:= (_max / (1024 * 1024)) | _max := format("%d", field=_max)//result in megabytes
| table([_max, dataspace, queryID, query])

#type=humio #kind=metrics name=load-segment-total |  timechart(#vhost, function=max(m1), limit=100, minSpan=1m)

#type=humio #kind=logs class=c.h.j.RunningQueriesLoggerJob "Highest Cost query"
| repoUser:= format("%s/%s", field=[dataspace, initiatingUser])
| top(repoUser, sum=deltaTotalCost, as=cost)
| table([cost, repoUser], sortby=cost)

$humio/insights:searchQueue()

#type=humio #kind=metrics name=/^query-millis\// | timechart(function={groupby([name, #vhost], function=max(m1)) | sum(_max)})

#type=humio #kind=logs class="*QuerySessions*" "query was restarted" | timechart(reason)

#type=humio #kind=requests | uri=/\/(?<repo>[^\/]+)\/queryjobs$/ internal=false | timechart(repo, limit=100, minSpan=1m, unit="1/m")

#type=humio #kind=metrics name=local-query-segments-queue*
| timechart(name, function={groupby(#vhost, function=max(value)) | sum(_max)}, limit=100, minSpan=1m)

#type=humio #kind=metrics name=time-livequery | timechart(function={groupby(#vhost, function=max(value)) | sum(_max)})

#type=humio #kind=metrics name=read-compressed-bytes  metric_type=METER | timechart(#vhost, limit=100, minSpan=1m, function=max(m1))

#type=humio #kind=nonsensitive name="stat_tasks" | time:=(utime+stime) | timechart(threadGroup, function=avg(time, as=ticks), limit=50, minSpan=1m)

$humio/insights:slowWarningsInQueries()

#type=humio #kind=metrics name=livequery-count | timechart(#vhost, function=max(value), limit=100, minSpan=1m)

#type=humio #kind=logs class=c.h.j.RunningQueriesLoggerJob "Highest Cost query"
| top(queryInput, sum=deltaTotalCost, as=cost)
| table([cost, queryInput], sortby=cost)

#type=humio #kind=logs | class = "*QuerySessions" "creating new query" | join({#type=humio #kind=threaddumps state=RUNNABLE  group=query | top(query)}, key=query, field=queryID, include=[_count]) | repository:= rename(dataspace) | table([_count, repository, queryID, query], sortby=_count)

#type=humio  #kind=threaddumps state=RUNNABLE | timechart(query, limit=100, minSpan=10s)

#type=humio #kind=metrics name="query-static-delta-cpu-usage"
| timechart(function={groupby(#vhost, function=max(mean)) | sum(_max, as=mean)})

#type=humio #kind=requests | uri=/\/query\/?$|queryjobs\/?$/ method=POST internal=false | timechart(#vhost, limit=100, minSpan=1m, unit="1/m")

#type=humio #kind=requests | uri=/\/query$/ internal=false | timechart(#vhost, limit=100, minSpan=1m, unit="1/m")

#type=humio #kind=requests | uri=/\/(?<repo>[^\/]+)\/query$/ internal=false | timechart(repo, limit=100, minSpan=1m, unit="1/m")

#type=humio #kind=requests | uri=/\/query|queryjob/ internal=true | timechart(#vhost, limit=100, minSpan=1m, unit="1/s")

#type=humio #kind=metrics name="query-delta-total-cost"
| timechart(function={groupby(#vhost, function=max(mean)) | sum(_max, as=mean)})

#type=humio #kind=metrics name="query-live-delta-cpu-usage"
| timechart(function={groupby(#vhost, function=max(mean)) | sum(_max, as=mean)})

#type=humio #kind=metrics name=target-segment-compressed-size | median>0
| timechart(function=[avg(median, as=median), avg(p75, as=p75), avg(p95, as=p95)])

#kind=logs | "missing segment" "due to no currenthosts and no bucketId" segment=*
| class = c.h.c.ClusterManagement alerting=true dataspace!=humio-*
| groupby([dataspace, dataspaceId, segment], function=[max("@timestamp", as=lastDetected)], limit=max)
| @timestamp := lastDetected // trick: assign to @timestamp to get date/time formatted into dashboard timezone
| drop([lastDetected])

#type=humio #kind=metrics name=segment-merge-cpu-time | median > 0
| timechart(function=[avg(p75, as=p75), avg(median, as=median), avg(p95, as=p95)], minSpan=1m)

#type=humio #kind=metrics name=segment-merge-latency-ms
| timechart(function=[avg(median, as=median), avg(p75, as=p75), avg(p95, as=p95)])

#type=humio #kind=metrics name=segment-merge-cpu-time
| timechart(#vhost, function=avg(m1), limit=100, minSpan=1m, unit="1/s to 1/hour")

#type=humio #kind=metrics name=segment-merge-cpu-time mean>0
| timechart(#vhost, function=[avg(m1, as=rate), avg(mean, as=average_time)], limit=100, minSpan=1m, unit="1/s to 1/hour")
| time:= (average_time * rate / 1000000000) | drop([average_time, rate])

#type=humio #kind=metrics name=target-segment-blocks median>0
| timechart(function=[avg(median, as=median), avg(p75, as=p75), avg(p95, as=p95)])

$humio/insights:globalSnapshotSize()

#type=humio #kind=metrics | name=/^datasource-count(\/[^\/]+)?\/(?<repo>.*)$/
| value>0 | timechart(repo, function={groupby(name, function=max(value))|sum(_max)}, limit=100)