Prometheus TSDB

Oldest record in Prometheus

cd /home/costa/data/prometheus-data ;grep -r minTime --include meta.json | awk '{print $3}' | sort | head -n1 | rev| cut -c5- | rev | { read gmt ; date -d "@$gmt" ; }

snapshot https://gist.github.com/ksingh7/d5e4414d92241e0802e59fa4c585b98b

How to drop and delete metrics in Prometheus https://faun.pub/how-to-drop-and-delete-metrics-in-prometheus-7f5e6911fb33

Find all unused metrics

mimirtool installation

curl -L -o mimirtool https://github.com/grafana/mimir/releases/latest/download/mimirtool-linux-amd64
chmod +x mimirtool
sudo mv mimirtool /usr/local/bin/mimirtool

Grafana token

glsa_20cNpxZw8mXIqebza9wk7jcFKm3KfZFk_2efbc31a

curl -H "Authorization: Bearer glsa_20cNpxZw8mXIqebza9wk7jcFKm3KfZFk_2efbc31a" http://10.0.0.200:30725/api/dashboards/home

mimirtool analyze grafana --address=http://10.0.0.200:30725 --key="glsa_20cNpxZw8mXIqebza9wk7jcFKm3KfZFk_2efbc31a"
mimirtool analyze prometheus --address=http://10.0.0.200:30900
jq -r ".in_use_metric_counts[].metric" prometheus-metrics.json | sort > used_metrics.txt
jq -r ".additional_metric_counts[].metric" prometheus-metrics.json | sort > unused_metrics.txt

Delete unused metrics

<e.g.> curl -X POST -g http://10.0.0.200:30900/api/v1/admin/tsdb/delete_series?match[]=workqueue_adds_total
n=1 ; for i in `cat unused_metrics.txt`; do echo "$((n++)) $i" ; curl -X POST -g 'http://10.0.0.200:30900/api/v1/admin/tsdb/delete_series?match[]='$i ; done
curl -XPOST 'http://10.0.0.200:30900/api/v1/admin/tsdb/clean_tombstones'

For reverse proxy:

 curl -H "Authorization: Bearer glsa_20cNpxZw8mXIqebza9wk7jcFKm3KfZFk_2efbc31a" \
  http://10.0.0.200/grafana/api/dashboards/home
 
 mimirtool analyze grafana \
  --address=http://10.0.0.200/grafana \
  --key="glsa_20cNpxZw8mXIqebza9wk7jcFKm3KfZFk_2efbc31a"
 
 mimirtool analyze prometheus \
  --address=http://10.0.0.200/prometheus
 
 jq -r ".in_use_metric_counts[].metric" prometheus-metrics.json | sort > used_metrics.txt
 jq -r ".additional_metric_counts[].metric" prometheus-metrics.json | sort > unused_metrics.txt
 
 # Delete unused metrics
 
 curl -X POST -g 'http://10.0.0.200/prometheus/api/v1/admin/tsdb/delete_series?match[]=workqueue_adds_total'
 
 n=1 ; for i in $(cat unused_metrics.txt); do   echo "$((n++)) $i" ; curl -X POST -g "http://10.0.0.200/prometheus/api/v1/admin/tsdb/delete_series?match[]=$i" ; done
 
 curl -X POST http://10.0.0.200/prometheus/api/v1/admin/tsdb/clean_tombstones