Expose rate limit bucket tokens as metric (#502)

server/configs.go

@@ -1,5 +1,7 @@
 package server
 
+import "time"
+
 type WebhookConfig struct {
 	Url         string `usage:"If set, a POST request that contains connection status notifications will be sent to this HTTP address"`
 	RequireUser bool   `default:"false" usage:"Indicates if the webhook will only be called if a user is connecting rather than just server list/ping"`
@@ -24,30 +26,31 @@ type NgrokConfig struct {
 }
 
 type Config struct {
-	Port                  int               `default:"25565" usage:"The [port] bound to listen for Minecraft client connections"`
+	Port                   int               `default:"25565" usage:"The [port] bound to listen for Minecraft client connections"`
-	Default               string            `usage:"host:port of a default Minecraft server to use when mapping not found"`
+	Default                string            `usage:"host:port of a default Minecraft server to use when mapping not found"`
-	Mapping               map[string]string `usage:"Comma or newline delimited or repeated mappings of externalHostname=host:port"`
+	Mapping                map[string]string `usage:"Comma or newline delimited or repeated mappings of externalHostname=host:port"`
-	ApiBinding            string            `usage:"The [host:port] bound for servicing API requests"`
+	ApiBinding             string            `usage:"The [host:port] bound for servicing API requests"`
-	CpuProfile            string            `usage:"Enables CPU profiling and writes to given path"`
+	CpuProfile             string            `usage:"Enables CPU profiling and writes to given path"`
-	ConnectionRateLimit   int               `default:"1" usage:"Max number of connections to allow per second"`
+	ConnectionRateLimit    int               `default:"1" usage:"Max number of connections to allow per second"`
-	InKubeCluster         bool              `usage:"Use in-cluster Kubernetes config"`
+	InKubeCluster          bool              `usage:"Use in-cluster Kubernetes config"`
-	KubeConfig            string            `usage:"The path to a Kubernetes configuration file"`
+	KubeConfig             string            `usage:"The path to a Kubernetes configuration file"`
-	KubeNamespace         string            `usage:"The namespace to watch or blank for all, which is the default"`
+	KubeNamespace          string            `usage:"The namespace to watch or blank for all, which is the default"`
-	InDocker              bool              `usage:"Use Docker service discovery"`
+	InDocker               bool              `usage:"Use Docker service discovery"`
-	InDockerSwarm         bool              `usage:"Use Docker Swarm service discovery"`
+	InDockerSwarm          bool              `usage:"Use Docker Swarm service discovery"`
-	DockerSocket          string            `default:"unix:///var/run/docker.sock" usage:"Path to Docker socket to use"`
+	DockerSocket           string            `default:"unix:///var/run/docker.sock" usage:"Path to Docker socket to use"`
-	DockerTimeout         int               `default:"0" usage:"Timeout configuration in seconds for the Docker integrations"`
+	DockerTimeout          int               `default:"0" usage:"Timeout configuration in seconds for the Docker integrations"`
-	DockerRefreshInterval int               `default:"15" usage:"Refresh interval in seconds for the Docker integrations"`
+	DockerRefreshInterval  int               `default:"15" usage:"Refresh interval in seconds for the Docker integrations"`
-	DockerApiVersion      string            `usage:"Instead of auto-negotiating, use specific Docker API version"`
+	DockerApiVersion       string            `usage:"Instead of auto-negotiating, use specific Docker API version"`
-	MetricsBackend        string            `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb,prometheus"`
+	MetricsBackend         string            `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb,prometheus"`
-	MetricsBackendConfig  MetricsBackendConfig
+	MetricsBackendConfig   MetricsBackendConfig
-	UseProxyProtocol      bool     `default:"false" usage:"Send PROXY protocol to backend servers"`
+	MetricsRateLimitPeriod time.Duration `default:"1s" usage:"The period at which the rate limit bucket's metrics are set: 0 to disable (default 1s)"`
-	ReceiveProxyProtocol  bool     `default:"false" usage:"Receive PROXY protocol from backend servers, by default trusts every proxy header that it receives, combine with -trusted-proxies to specify a list of trusted proxies"`
+	UseProxyProtocol       bool          `default:"false" usage:"Send PROXY protocol to backend servers"`
-	TrustedProxies        []string `usage:"Comma delimited list of CIDR notation IP blocks to trust when receiving PROXY protocol"`
+	ReceiveProxyProtocol   bool          `default:"false" usage:"Receive PROXY protocol from backend servers, by default trusts every proxy header that it receives, combine with -trusted-proxies to specify a list of trusted proxies"`
-	RecordLogins          bool     `default:"false" usage:"Log and generate metrics on player logins. Metrics only supported with influxdb or prometheus backend"`
+	TrustedProxies         []string      `usage:"Comma delimited list of CIDR notation IP blocks to trust when receiving PROXY protocol"`
-	Routes                RoutesConfig
+	RecordLogins           bool          `default:"false" usage:"Log and generate metrics on player logins. Metrics only supported with influxdb or prometheus backend"`
-	Ngrok                 NgrokConfig
+	Routes                 RoutesConfig
-	AutoScale             AutoScale
+	Ngrok                  NgrokConfig
+	AutoScale              AutoScale
 
 	ClientsToAllow []string `usage:"Zero or more client IP addresses or CIDRs to allow. Takes precedence over deny."`
 	ClientsToDeny  []string `usage:"Zero or more client IP addresses or CIDRs to deny. Ignored if any configured to allow"`

server/connector.go

@@ -111,13 +111,13 @@ func (c *Connector) UseClientFilter(filter *ClientFilter) {
 	c.clientFilter = filter
 }
 
-func (c *Connector) StartAcceptingConnections(listenAddress string, connRateLimit int) error {
+func (c *Connector) StartAcceptingConnections(listenAddress string, connRateLimit int, metricsPeriod time.Duration) error {
 	ln, err := c.createListener(listenAddress)
 	if err != nil {
 		return err
 	}
 
-	go c.acceptConnections(ln, connRateLimit)
+	go c.acceptConnections(ln, connRateLimit, metricsPeriod)
 
 	return nil
 }
@@ -201,11 +201,14 @@ func (c *Connector) AcceptConnection(conn net.Conn) {
 	go c.HandleConnection(conn)
 }
 
-func (c *Connector) acceptConnections(ln net.Listener, connRateLimit int) {
+func (c *Connector) acceptConnections(ln net.Listener, connRateLimit int, metricsPeriod time.Duration) {
 	//noinspection GoUnhandledErrorResult
 	defer ln.Close()
 
 	bucket := ratelimit.NewBucketWithRate(float64(connRateLimit), int64(connRateLimit*2))
+	if metricsPeriod > 0 {
+		go c.bucketMetrics(bucket, metricsPeriod)
+	}
 
 	for {
 		select {
@@ -223,6 +226,19 @@ func (c *Connector) acceptConnections(ln net.Listener, connRateLimit int) {
 	}
 }
 
+func (c *Connector) bucketMetrics(bucket *ratelimit.Bucket, period time.Duration) {
+	ticker := time.NewTicker(period)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-c.ctx.Done():
+			return
+		case <-ticker.C:
+			c.metrics.RateLimitAvailable.Set(float64(bucket.Available()))
+		}
+	}
+}
+
 func (c *Connector) HandleConnection(frontendConn net.Conn) {
 	c.metrics.ConnectionsFrontend.Add(1)
 	//noinspection GoUnhandledErrorResult

server/metrics.go

@@ -78,6 +78,7 @@ type ConnectorMetrics struct {
 	ServerActivePlayer      metrics.Gauge
 	ServerLogins            metrics.Counter
 	ServerActiveConnections metrics.Gauge
+	RateLimitAvailable      metrics.Gauge
 }
 
 func (b expvarMetricsBuilder) BuildConnectorMetrics() *ConnectorMetrics {
@@ -91,6 +92,7 @@ func (b expvarMetricsBuilder) BuildConnectorMetrics() *ConnectorMetrics {
 		ServerActivePlayer:      expvarMetrics.NewGauge("server_active_player"),
 		ServerLogins:            expvarMetrics.NewCounter("server_logins"),
 		ServerActiveConnections: expvarMetrics.NewGauge("server_active_connections"),
+		RateLimitAvailable:      expvarMetrics.NewGauge("rate_limit_available"),
 	}
 }
 
@@ -112,6 +114,7 @@ func (b discardMetricsBuilder) BuildConnectorMetrics() *ConnectorMetrics {
 		ServerActivePlayer:      discardMetrics.NewGauge(),
 		ServerLogins:            discardMetrics.NewCounter(),
 		ServerActiveConnections: discardMetrics.NewGauge(),
+		RateLimitAvailable:      discardMetrics.NewGauge(),
 	}
 }
 
@@ -164,6 +167,7 @@ func (b *influxMetricsBuilder) BuildConnectorMetrics() *ConnectorMetrics {
 		ServerActivePlayer:      metrics.NewGauge("mc_router_server_player_active"),
 		ServerLogins:            metrics.NewCounter("mc_router_server_logins"),
 		ServerActiveConnections: metrics.NewGauge("mc_router_server_active_connections"),
+		RateLimitAvailable:      metrics.NewGauge("mc_router_rate_limit_available"),
 	}
 }
 
@@ -225,5 +229,10 @@ func (b prometheusMetricsBuilder) BuildConnectorMetrics() *ConnectorMetrics {
 			Name:      "server_active_connections",
 			Help:      "The number of active connections per server",
 		}, []string{"server_address"})),
+		RateLimitAvailable: prometheusMetrics.NewGauge(promauto.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "mc_router",
+			Name:      "rate_limit_available",
+			Help:      "The number of available tokens in the rate limit bucket",
+		}, nil)),
 	}
 }

server/server.go

@@ -202,6 +202,7 @@ func (s *Server) Run() {
 	err := s.connector.StartAcceptingConnections(
 		net.JoinHostPort("", strconv.Itoa(s.config.Port)),
 		s.config.ConnectionRateLimit,
+		s.config.MetricsRateLimitPeriod,
 	)
 	if err != nil {
 		logrus.WithError(err).Error("Could not start accepting connections")