Refactored server setup and run out of main (#425)

2025-07-05 19:32:33 -05:00
parent 1ee3eb4de3
commit 05c57c3b85
6 changed files with 308 additions and 223 deletions
@@ -0,0 +1,50 @@
+package server
+
+type WebhookConfig struct {
+	Url         string `usage:"If set, a POST request that contains connection status notifications will be sent to this HTTP address"`
+	RequireUser bool   `default:"false" usage:"Indicates if the webhook will only be called if a user is connecting rather than just server list/ping"`
+}
+
+type AutoScale struct {
+	Up        bool   `usage:"Increase Kubernetes StatefulSet Replicas (only) from 0 to 1 on respective backend servers when accessed"`
+	Down      bool   `default:"false" usage:"Decrease Kubernetes StatefulSet Replicas (only) from 1 to 0 on respective backend servers after there are no connections"`
+	DownAfter string `default:"10m" usage:"Server scale down delay after there are no connections"`
+	AllowDeny string `usage:"Path to config for server allowlists and denylists. If a global/server entry is specified, only players allowed to connect to the server will be able to trigger a scale up when -auto-scale-up is enabled or cancel active down scalers when -auto-scale-down is enabled"`
+}
+
+type RoutesConfig struct {
+	Config      string `usage:"Name or full [path] to routes config file"`
+	ConfigWatch bool   `usage:"Watch for config file changes"`
+}
+
+type Config struct {
+	Port                  int               `default:"25565" usage:"The [port] bound to listen for Minecraft client connections"`
+	Default               string            `usage:"host:port of a default Minecraft server to use when mapping not found"`
+	Mapping               map[string]string `usage:"Comma or newline delimited or repeated mappings of externalHostname=host:port"`
+	ApiBinding            string            `usage:"The [host:port] bound for servicing API requests"`
+	CpuProfile            string            `usage:"Enables CPU profiling and writes to given path"`
+	ConnectionRateLimit   int               `default:"1" usage:"Max number of connections to allow per second"`
+	InKubeCluster         bool              `usage:"Use in-cluster Kubernetes config"`
+	KubeConfig            string            `usage:"The path to a Kubernetes configuration file"`
+	InDocker              bool              `usage:"Use Docker service discovery"`
+	InDockerSwarm         bool              `usage:"Use Docker Swarm service discovery"`
+	DockerSocket          string            `default:"unix:///var/run/docker.sock" usage:"Path to Docker socket to use"`
+	DockerTimeout         int               `default:"0" usage:"Timeout configuration in seconds for the Docker integrations"`
+	DockerRefreshInterval int               `default:"15" usage:"Refresh interval in seconds for the Docker integrations"`
+	MetricsBackend        string            `default:"discard" usage:"Backend to use for metrics exposure/publishing: discard,expvar,influxdb,prometheus"`
+	MetricsBackendConfig  MetricsBackendConfig
+	UseProxyProtocol      bool     `default:"false" usage:"Send PROXY protocol to backend servers"`
+	ReceiveProxyProtocol  bool     `default:"false" usage:"Receive PROXY protocol from backend servers, by default trusts every proxy header that it receives, combine with -trusted-proxies to specify a list of trusted proxies"`
+	TrustedProxies        []string `usage:"Comma delimited list of CIDR notation IP blocks to trust when receiving PROXY protocol"`
+	RecordLogins          bool     `default:"false" usage:"Log and generate metrics on player logins. Metrics only supported with influxdb or prometheus backend"`
+	Routes                RoutesConfig
+	NgrokToken            string `usage:"If set, an ngrok tunnel will be established. It is HIGHLY recommended to pass as an environment variable."`
+	AutoScale             AutoScale
+
+	ClientsToAllow []string `usage:"Zero or more client IP addresses or CIDRs to allow. Takes precedence over deny."`
+	ClientsToDeny  []string `usage:"Zero or more client IP addresses or CIDRs to deny. Ignored if any configured to allow"`
+
+	SimplifySRV bool `default:"false" usage:"Simplify fully qualified SRV records for mapping"`
+
+	Webhook WebhookConfig `usage:"Webhook configuration"`
+}
@@ -49,6 +49,10 @@ func (r *routesConfigLoader) Load(routesConfigFileName string) error {
 }

 func (r *routesConfigLoader) Reload() error {
+	if !r.isEnabled() {
+		return nil
+	}
+
 	config, readErr := r.readFile()

 	if readErr != nil {
@@ -0,0 +1,210 @@
+package server
+
+import (
+	"context"
+	"fmt"
+	"github.com/sirupsen/logrus"
+	"net"
+	"os"
+	"runtime/pprof"
+	"strconv"
+	"time"
+)
+
+type Server struct {
+	ctx              context.Context
+	config           *Config
+	connector        *Connector
+	reloadConfigChan chan struct{}
+	doneChan         chan struct{}
+}
+
+func NewServer(ctx context.Context, config *Config) (*Server, error) {
+	if config.CpuProfile != "" {
+		cpuProfileFile, err := os.Create(config.CpuProfile)
+		if err != nil {
+			return nil, fmt.Errorf("could not create cpu profile file: %w", err)
+		}
+		//goland:noinspection GoUnhandledErrorResult
+		defer cpuProfileFile.Close()
+
+		logrus.WithField("file", config.CpuProfile).Info("Starting cpu profiling")
+		err = pprof.StartCPUProfile(cpuProfileFile)
+		if err != nil {
+			return nil, fmt.Errorf("could not start cpu profile: %w", err)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
+	var err error
+
+	var autoScaleAllowDenyConfig *AllowDenyConfig = nil
+	if config.AutoScale.AllowDeny != "" {
+		autoScaleAllowDenyConfig, err = ParseAllowDenyConfig(config.AutoScale.AllowDeny)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse autoscale allow-deny-list: %w", err)
+		}
+	}
+
+	metricsBuilder := NewMetricsBuilder(config.MetricsBackend, &config.MetricsBackendConfig)
+
+	downScalerEnabled := config.AutoScale.Down && (config.InKubeCluster || config.KubeConfig != "")
+	downScalerDelay, err := time.ParseDuration(config.AutoScale.DownAfter)
+	if err != nil {
+		return nil, fmt.Errorf("could not parse auto-scale-down-after duration: %w", err)
+	}
+	// Only one instance should be created
+	DownScaler = NewDownScaler(ctx, downScalerEnabled, downScalerDelay)
+
+	if config.Routes.Config != "" {
+		err := RoutesConfigLoader.Load(config.Routes.Config)
+		if err != nil {
+			return nil, fmt.Errorf("could not load routes config file: %w", err)
+		}
+
+		if config.Routes.ConfigWatch {
+			err := RoutesConfigLoader.WatchForChanges(ctx)
+			if err != nil {
+				return nil, fmt.Errorf("could not watch for changes to routes config file: %w", err)
+			}
+		}
+	}
+
+	Routes.RegisterAll(config.Mapping)
+	if config.Default != "" {
+		Routes.SetDefaultRoute(config.Default)
+	}
+
+	if config.ConnectionRateLimit < 1 {
+		config.ConnectionRateLimit = 1
+	}
+
+	trustedIpNets := make([]*net.IPNet, 0)
+	for _, ip := range config.TrustedProxies {
+		_, ipNet, err := net.ParseCIDR(ip)
+		if err != nil {
+			return nil, fmt.Errorf("could not parse trusted proxy CIDR block: %w", err)
+		}
+		trustedIpNets = append(trustedIpNets, ipNet)
+	}
+
+	connector := NewConnector(metricsBuilder.BuildConnectorMetrics(), config.UseProxyProtocol, config.ReceiveProxyProtocol, trustedIpNets, config.RecordLogins, autoScaleAllowDenyConfig)
+
+	clientFilter, err := NewClientFilter(config.ClientsToAllow, config.ClientsToDeny)
+	if err != nil {
+		return nil, fmt.Errorf("could not create client filter: %w", err)
+	}
+	connector.SetClientFilter(clientFilter)
+
+	if config.Webhook.Url != "" {
+		logrus.WithField("url", config.Webhook.Url).
+			WithField("require-user", config.Webhook.RequireUser).
+			Info("Using webhook for connection status notifications")
+		connector.SetConnectionNotifier(
+			NewWebhookNotifier(config.Webhook.Url, config.Webhook.RequireUser))
+	}
+
+	if config.NgrokToken != "" {
+		connector.UseNgrok(config.NgrokToken)
+	}
+
+	if config.ApiBinding != "" {
+		StartApiServer(config.ApiBinding)
+	}
+
+	if config.InKubeCluster {
+		err = K8sWatcher.StartInCluster(config.AutoScale.Up, config.AutoScale.Down)
+		if err != nil {
+			return nil, fmt.Errorf("could not start in-cluster k8s integration: %w", err)
+		} else {
+			defer K8sWatcher.Stop()
+		}
+	} else if config.KubeConfig != "" {
+		err := K8sWatcher.StartWithConfig(config.KubeConfig, config.AutoScale.Up, config.AutoScale.Down)
+		if err != nil {
+			return nil, fmt.Errorf("could not start k8s integration with kube config: %w", err)
+		} else {
+			defer K8sWatcher.Stop()
+		}
+	}
+
+	if config.InDocker {
+		err = DockerWatcher.Start(config.DockerSocket, config.DockerTimeout, config.DockerRefreshInterval, config.AutoScale.Up, config.AutoScale.Down)
+		if err != nil {
+			return nil, fmt.Errorf("could not start docker integration: %w", err)
+		} else {
+			defer DockerWatcher.Stop()
+		}
+	}
+
+	if config.InDockerSwarm {
+		err = DockerSwarmWatcher.Start(config.DockerSocket, config.DockerTimeout, config.DockerRefreshInterval, config.AutoScale.Up, config.AutoScale.Down)
+		if err != nil {
+			return nil, fmt.Errorf("could not start docker swarm integration: %w", err)
+		} else {
+			defer DockerSwarmWatcher.Stop()
+		}
+	}
+
+	Routes.SimplifySRV(config.SimplifySRV)
+
+	err = metricsBuilder.Start(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("could not start metrics reporter: %w", err)
+	}
+
+	return &Server{
+		ctx:              ctx,
+		config:           config,
+		connector:        connector,
+		reloadConfigChan: make(chan struct{}),
+		doneChan:         make(chan struct{}),
+	}, nil
+}
+
+// Done provides a channel notified when the server has closed all connections, etc
+func (s *Server) Done() <-chan struct{} {
+	return s.doneChan
+}
+
+func (s *Server) notifyDone() {
+	s.doneChan <- struct{}{}
+}
+
+// ReloadConfig indicates that an external request, such as a SIGHUP,
+// is requesting the routes config file to be reloaded, if enabled
+func (s *Server) ReloadConfig() {
+	s.reloadConfigChan <- struct{}{}
+}
+
+// Run will run the server until the context is done or a fatal error occurs, so this should be
+// in a go routine.
+func (s *Server) Run() {
+	err := s.connector.StartAcceptingConnections(s.ctx,
+		net.JoinHostPort("", strconv.Itoa(s.config.Port)),
+		s.config.ConnectionRateLimit,
+	)
+	if err != nil {
+		logrus.WithError(err).Error("Could not start accepting connections")
+		s.notifyDone()
+		return
+	}
+
+	for {
+		select {
+		case <-s.reloadConfigChan:
+			if err := RoutesConfigLoader.Reload(); err != nil {
+				logrus.WithError(err).
+					Error("Could not re-read the routes config file")
+			}
+
+		case <-s.ctx.Done():
+			logrus.Info("Stopping. Waiting for connections to complete...")
+			s.connector.WaitForConnections()
+			logrus.Info("Stopped")
+			s.notifyDone()
+			return
+		}
+	}
+
+}