Custom Prometheus Exporter
Writing a custom Prometheus Exporter in Golang (to get better in Go and have some practical experience with Golang)
I installed Golang 1.23.3 in my windows PC and then init my first test exporter as following
PS C:\Users\wajiw\OneDrive\Desktop\Exporters> go mod init test_exporter
go: creating new go.mod: module test_exporter
PS C:\Users\wajiw\OneDrive\Desktop\Exporters> go get github.com/prometheus/client_golang
go: downloading github.com/prometheus/client_golang v1.20.5
go: added github.com/prometheus/client_golang v1.20.5
Just quick tip ifcouldn't import Golangissue happens in vscode. Thats probably due to vscode open in root rather than the actual ‘working’ directory where we are ‘working’. This can be solved using thego workcomand
PS C:\Users\wajiw\OneDrive\Desktop\Exporters> go work init
PS C:\Users\wajiw\OneDrive\Desktop\Exporters> go work use .\test_exporter\
Wrote the basic exporter entry point
package main
import (
"fmt"
"log"
"net/http"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
func init() {
fmt.Println("Exposing metrics under /metrics")
}
func main() {
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(":9101", nil))
}
Running the above
PS C:\Users\wajiw\OneDrive\Desktop\Exporters> go run .\test_exporter\main.go
Exposing metrics under /metrics
The metrics page for now
Password Expiry Exporter (Simple Exporter)
The main.go file
package main
import (
"log"
"net/http"
"os/exec"
"os"
"strings"
"time"
"flag"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
listenAddress = flag.String("web.listen-address", ":8700", "Address to Listen on for metrics")
)
type passwordExpiryCollector struct {
user string
node string
passwordExpiry *prometheus.Desc
}
func newPasswordExpiryCollector(user string) *passwordExpiryCollector {
node, _ := os.Hostname()
return &passwordExpiryCollector{
user: user,
node: node,
passwordExpiry: prometheus.NewDesc("password_expiry_days", "Days until password expiry", nil, prometheus.Labels{"user": user, "node": node}),
}
}
func (collector *passwordExpiryCollector) Describe(ch chan<- *prometheus.Desc){
ch <- collector.passwordExpiry
}
func (collector *passwordExpiryCollector) Collect(ch chan<- prometheus.Metric) {
var value float64
out, _ := exec.Command("chage", "-l", collector.user).Output()
lines := strings.Split(string(out), "\n")
for _, line := range lines {
if strings.HasPrefix(line, "Password expires") {
dateStr := strings.TrimSpace(strings.Split(line, ":")[1])
if dateStr == "never" {
value = float64(999999)
} else {
t, _ := time.Parse("Jan 2, 2006", dateStr)
value = float64(t.Sub(time.Now()).Hours() / 24)
}
}
}
ch <- prometheus.MustNewConstMetric(collector.passwordExpiry, prometheus.GaugeValue, value)
}
func main() {
var user string
flag.StringVar(&user, "u", "", "Username to check password expiry for")
flag.Parse()
if user == "" {
log.Fatal("set username using the '-u' flag")
}
userCollector := newPasswordExpiryCollector(user)
prometheus.MustRegister(userCollector)
log.Printf("Exposing Metrics under /metrics at port %s", *listenAddress)
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
Deploying the above as systemd
## First build the go code
$ go build -o password_expiry_exporter .
$ sudo cp -rp password_expiry_exporter /usr/bin
The systemd file
[Unit]
Description=Password Expiry Exporter
Wants=network-online.target
[Service]
User=root
ExecStart=/usr/bin/password_expiry_exporter -u waji
SyslogIdentifier=password_expiry_exporter
[Install]
WantedBy=multi-user.target
We should be able to scrape metrics via Local prometheus or if we want to scrape the above metrics from a k8s prometheus, we can create an Endpoint object as below (followed by a Service & ServiceMonitor objects too)
apiVersion: v1
kind: Endpoints
metadata:
name: password-expiry-exporter
labels:
app: password-expiry-exporter
namespace: monitoring
subsets:
- addresses:
- ip: "<The node IP>"
ports:
- name: metrics
port: 8700
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
labels:
app: password-expiry-exporter
name: password-expiry-exporter
namespace: monitoring
spec:
ports:
- name: metrics
port: 8700
protocol: TCP
selector:
app: password-expiry-exporter
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
app: password-expiry-exporter
name: password-expiry-exporter
namespace: monitoring
spec:
endpoints:
- intervals: 30s
port: metrics
scheme: http
jobLabel: app
selector:
matchLabels:
app: password-expiry-exporter
Storage Exporter (Total, Used, Free & Session)
We required a simple Exporter that used Dell’s Powerflex Storage API to get Total, Used, Free & Session Number for connected devices against the Storage.
go mod init powerflex_exporter
go get github.com/joho/godotenv
go get github.com/prometheus/client_golang/prometheus
As we are using godotenv
## Below values need to be set
cat auth
USERNAME=
PASSWORD=
The Dockerfile
FROM golang:1.16.6-alpiine3.14
RUN mkdir /app
COPY . /app
WORKDIR /app
RUN go build -o main .
CMD ["/app/main"]
Won’t include go.mod & go.sum files here
The main code
package main
import (
"log"
"net/http"
"flag"
"encoding/json"
"io/ioutil"
"crypto/tls"
"strings"
"os"
"github.com/joho/godotenv"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
var (
listenAddress = flag.String("web.listen-address", ":9070", "Address to Listen on for metrics")
accessTokens map[string]string
username string
password string
)
var storagePoolIDs = map[string]string{
//Included Storage Pool Gateway IPs and Storage Pool IDs alongwith their respective Pool Names
"1.1.1.1": {"f839f89d9023", "SP01"}
}
type Stats struct {
Total float64 `json:"netMaxUserDataCapacityInKb"`
Used float64 `json:"netUserDataCapacityInKb"`
Free float64 `json:"netUnusedCapacityInKb"`
}
type Session struct {
SdcGuid string `json:"sdcGuid"`
}
type powerflexCollector struct {
totalCapacityDesc *prometheus.Desc
usedCapacityDesc *prometheus.Desc
freeCapacityDesc *prometheus.Desc
sessionsDesc *prometheus.Desc
}
func newPowerflexCollector() *powerflexCollector {
return &powerflexCollector{
totalCapacityDesc: prometheus.NewDesc("powerflex_total_capacity",
"Total Capacity of storage pool", []string{"storagePool"}, nil,
),
usedCapacityDesc: prometheus.NewDesc("powerflex_used_capacity",
"Used Capacity of storage pool", []string{"storagePool"}, nil,
),
freeCapacityDesc: prometheus.NewDesc("powerflex_free_capacity",
"Free Capacity of storage pool", []string{"storagePool"}, nil,
),
sessionsDesc: prometheus.NewDesc("powerflex_session_count",
"Session Count for each Storage Pool", []string{"storagePool"}, nil,
),
}
}
func loginAndGetToken(gateWayIP string) (string, error) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", "https://"+gateWayIP+"/api/login", nil)
if err != nil {
return "", err
}
req.SetBasicAuth(username, password)
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
return strings.Trim(string(body), "\""), nil
}
func getStats(gateWayIP string, storagePoolID string, accessToken string) (Stats, error) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", "https://"+gateWayIP+"/api/instances/StoragePool::"+storagePoolID+"/relationships/Statistics", nil)
if err != nil {
return Stats{}, err
}
req.SetBasicAuth(username, accessToken)
resp, err := client.Do(req)
if err != nil {
return Stats{}, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return Stats {}, err
}
var stats Stats
json.Unmarshal(body, &stats)
return stats, nil
}
func getSessions(gateWayIP string, accessToken string) (int, error) {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", "https://"+gateWayIP+"/api/types/Sdc/instance", nil)
if err != nil {
return 0, err
}
req.SetBasicAuth(username, accessToken)
resp, err := client.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return 0, err
}
var sessions []Session
err = json.Unmarshal(body, &sessions)
if err != nil {
return 0, err
}
count := 0
for _, session := range sessions{
if session.SdcGuid != "" {
count ++
}
}
return count, nil
}
func (collector *powerflexCollector) Describe(ch chan<- *prometheus.Desc){
ch <- collector.totalCapacityDesc
ch <- collector.usedCapacityDesc
ch <- collector.freeCapacityDesc
ch <- collector.sessionsDesc
}
func (collector *powerflexCollector) Collect(ch chan<- prometheus.Metric){
if accessTokens == nil {
accessTokens = make(map[string]string)
}
for gateWayIP, values := range storagePoolIDs {
storagePoolID := values[0]
storagePool := values[1]
if _, ok := accessTokens[storagePoolID]; !ok {
accessToken, err := loginAndGetToken(gateWayIP)
if err != nil {
log.Printf("Failed to login to storage Pool: %v", err)
continue
}
accessTokens[storagePoolID] = accessToken
}
stats, err := getStats(gateWayIP, storagePoolID, accessTokens[storagePoolID])
if err != nil {
log.Printf("Failed to fetch stats from storage Pool: %v", err)
continue
}
sessions, err := getSessions(gateWayIP, accessToken[storagePoolID])
if err != nil {
log.Printf("Failed to fetch sessions from storage Pool: %v", err)
continue
}
ch <- prometheus.MustNewConstMetric(collector.totalCapacityDesc, prometheus.GaugeValue, stats.Total, storagePool)
ch <- prometheus.MustNewConstMetric(collector.usedCapacityDesc, prometheus.GaugeValue, stats.Used, storagePool)
ch <- prometheus.MustNewConstMetric(collector.freeCapacityDesc, prometheus.GaugeValue, stats.Free, storagePool)
ch <- prometheus.MustNewConstMetric(collector.sessionsDesc, prometheus.GaugeValue, float64(sessions), storagePool)
}
}
func init() {
err := godotenv.Load("auth")
if err != nil {
log.Fatal("Error loading env")
}
username = os.Getenv("USERNAME")
password = os.Getenv("PASSWORD")
}
func main() {
prometheus.MustRegister(newPowerflexCollector())
log.Printf("Exposing metrics under /metrics at port %s", *listenAddress)
http.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(*listenAddress, nil))
}