I'm trying to implement a docker monitoring solution in my organization, which will be used for internal monitoring. The problem is the box which is used, does not allow any http port to be exposed due to security reasons and also does not allow to assign networks to any of the containers. As a result, I'm unable to go forward with tools such as cadvisor etc. The only thing working for me is the docker stats command.
Now using this docker stats through API, I want to convert those metrics into a format which is supported by the prometheus, but I'm not sure on how to proceed with this. Once the metrics have been converted, using curl I am planning to push the metrics to a push gateway.
Can someone help me on how to convert this docker stats metrics to prometheus compatible metrics. I will be mainly needing the cpu, memory and disk utilization
P.S: I have heard about the prometheus python client API, but not sure if it can be used in this situation. Any lead on this would be helpful.
Sample docker API output:
-bash-5.0# curl --unix-socket /var/run/docker.sock "http://localhost/v1.40/containers/7148f8ea6610/stats?stream=false" |jq
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 3045 0 3045 0 0 1638 0 --:--:-- 0:00:01 --:--:-- 1637
{
"read": "2022-04-26T11:25:46.419629339Z",
"preread": "2022-04-26T11:25:45.416384103Z",
"pids_stats": {
"current": 169
},
"blkio_stats": {
"io_service_bytes_recursive": [
{
"major": 7,
"minor": 0,
"op": "Read",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Write",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Sync",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Async",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Discard",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Total",
"value": 0
},
{
"major": 8,
"minor": 0,
"op": "Read",
"value": 11395072
},
{
"major": 8,
"minor": 0,
"op": "Write",
"value": 12288
},
{
"major": 8,
"minor": 0,
"op": "Sync",
"value": 11407360
},
{
"major": 8,
"minor": 0,
"op": "Async",
"value": 0
},
{
"major": 8,
"minor": 0,
"op": "Discard",
"value": 0
},
{
"major": 8,
"minor": 0,
"op": "Total",
"value": 11407360
}
],
"io_serviced_recursive": [
{
"major": 7,
"minor": 0,
"op": "Read",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Write",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Sync",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Async",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Discard",
"value": 0
},
{
"major": 7,
"minor": 0,
"op": "Total",
"value": 0
},
{
"major": 8,
"minor": 0,
"op": "Read",
"value": 403
},
{
"major": 8,
"minor": 0,
"op": "Write",
"value": 2
},
{
"major": 8,
"minor": 0,
"op": "Sync",
"value": 405
},
{
"major": 8,
"minor": 0,
"op": "Async",
"value": 0
},
{
"major": 8,
"minor": 0,
"op": "Discard",
"value": 0
},
{
"major": 8,
"minor": 0,
"op": "Total",
"value": 405
}
],
"io_queue_recursive": [],
"io_service_time_recursive": [],
"io_wait_time_recursive": [],
"io_merged_recursive": [],
"io_time_recursive": [],
"sectors_recursive": []
},
"num_procs": 0,
"storage_stats": {},
"cpu_stats": {
"cpu_usage": {
"total_usage": 353478838223345,
"percpu_usage": [
177851824458947,
175627013764398
],
"usage_in_kernelmode": 136092500000000,
"usage_in_usermode": 213417730000000
},
"system_cpu_usage": 2619375300000000,
"online_cpus": 2,
"throttling_data": {
"periods": 0,
"throttled_periods": 0,
"throttled_time": 0
}
},
"precpu_stats": {
"cpu_usage": {
"total_usage": 353478690297736,
"percpu_usage": [
177851810524667,
175626879773069
],
"usage_in_kernelmode": 136092470000000,
"usage_in_usermode": 213417620000000
},
"system_cpu_usage": 2619373310000000,
"online_cpus": 2,
"throttling_data": {
"periods": 0,
"throttled_periods": 0,
"throttled_time": 0
}
},
"memory_stats": {
"usage": 7378079744,
"max_usage": 7715459072,
"stats": {
"active_anon": 270336,
"active_file": 2887680,
"cache": 5001216,
"dirty": 0,
"hierarchical_memory_limit": 9223372036854772000,
"hierarchical_memsw_limit": 9223372036854772000,
"inactive_anon": 2027520,
"inactive_file": 2027520,
"mapped_file": 135168,
"pgfault": 407636823,
"pgmajfault": 33,
"pgpgin": 219693705,
"pgpgout": 219692076,
"rss": 2027520,
"rss_huge": 0,
"total_active_anon": 1898262528,
"total_active_file": 2912403456,
"total_cache": 4865912832,
"total_dirty": 270336,
"total_inactive_anon": 454504448,
"total_inactive_file": 1874886656,
"total_mapped_file": 324538368,
"total_pgfault": 16369125632,
"total_pgmajfault": 13906,
"total_pgpgin": 7802786212,
"total_pgpgout": 7801043002,
"total_rss": 2273120256,
"total_rss_huge": 0,
"total_unevictable": 1351680,
"total_writeback": 540672,
"unevictable": 0,
"writeback": 0
},
"limit": 16522530816
},
"name": "/sample",
"id": "7148f8ea6610977655a86cad7d4d66103310b8381bfe3fdd6a29235deed43d19"
}
I found a github project which does exactly what you need. Maybe take a look at their source/solution.