enhance(upstream): proxy target parsing from multiple configs

Author: 0xJacky

api/analytic/nodes.go

@@ -65,8 +65,6 @@ func GetNodeStat(c *gin.Context) {
 					NodeInfo: nodeInfo,
 					NodeStat: stat,
 				}
-
-				logger.Debugf("Sending complete node info including version: %s", ver.Version)
 			}
 		} else {
 			// Send only stat information for performance

internal/cache/index.go

@@ -18,6 +18,9 @@ import (
 // ScanCallback is called during config scanning with file path and content
 type ScanCallback func(configPath string, content []byte) error
 
+// PostScanCallback is called after all files have been scanned (for second pass processing)
+type PostScanCallback func() error
+
 // Scanner watches and scans nginx config files
 type Scanner struct {
 	ctx        context.Context
@@ -28,10 +31,12 @@ type Scanner struct {
 }
 
 var (
-	scanner            *Scanner
-	scannerInitMutex   sync.Mutex
-	scanCallbacks      = make([]ScanCallback, 0)
-	scanCallbacksMutex sync.RWMutex
+	scanner                *Scanner
+	scannerInitMutex       sync.Mutex
+	scanCallbacks          = make([]ScanCallback, 0)
+	scanCallbacksMutex     sync.RWMutex
+	postScanCallbacks      = make([]PostScanCallback, 0)
+	postScanCallbacksMutex sync.RWMutex
 )
 
 // InitScanner initializes the config scanner
@@ -90,6 +95,13 @@ func RegisterCallback(callback ScanCallback) {
 	scanCallbacks = append(scanCallbacks, callback)
 }
 
+// RegisterPostScanCallback adds a callback to be executed after all files are scanned
+func RegisterPostScanCallback(callback PostScanCallback) {
+	postScanCallbacksMutex.Lock()
+	defer postScanCallbacksMutex.Unlock()
+	postScanCallbacks = append(postScanCallbacks, callback)
+}
+
 // Initialize sets up the scanner and starts watching
 func (s *Scanner) Initialize(ctx context.Context) error {
 	watcher, err := fsnotify.NewWatcher()
@@ -308,15 +320,16 @@ func (s *Scanner) executeCallbacks(filePath string, content []byte) {
 	}
 }
 
-// ScanAllConfigs scans all nginx configuration files
+// ScanAllConfigs scans all nginx configuration files with two-pass approach
 func (s *Scanner) ScanAllConfigs() error {
 	s.setScanningState(true)
 	defer s.setScanningState(false)
 
 	root := nginx.GetConfPath()
 
-	// Scan all files in the config directory and subdirectories
-	return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+	// First pass: scan all files to collect upstream definitions and other metadata
+	logger.Debug("ScanAllConfigs: Starting first pass (upstream definitions)")
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -329,12 +342,38 @@ func (s *Scanner) ScanAllConfigs() error {
 		// Only process regular files
 		if !d.IsDir() {
 			if err := s.scanSingleFile(path); err != nil {
-				logger.Error("Failed to scan config:", path, err)
+				logger.Error("Failed to scan config in first pass:", path, err)
 			}
 		}
 
 		return nil
 	})
+
+	if err != nil {
+		return err
+	}
+
+	// Second pass: parse proxy targets now that all upstreams are collected
+	logger.Debug("ScanAllConfigs: Starting second pass (proxy targets)")
+	return s.triggerProxyTargetsParsing()
+}
+
+// triggerProxyTargetsParsing triggers the second pass for proxy target parsing
+func (s *Scanner) triggerProxyTargetsParsing() error {
+	// Execute all post-scan callbacks
+	postScanCallbacksMutex.RLock()
+	callbacks := make([]PostScanCallback, len(postScanCallbacks))
+	copy(callbacks, postScanCallbacks)
+	postScanCallbacksMutex.RUnlock()
+
+	for _, callback := range callbacks {
+		if err := callback(); err != nil {
+			logger.Error("Post-scan callback failed:", err)
+			return err
+		}
+	}
+
+	return nil
 }
 
 // Shutdown cleans up scanner resources

internal/cron/upstream_availability.go

@@ -3,7 +3,6 @@ package cron
 import (
 	"time"
 
-	apiUpstream "github.com/0xJacky/Nginx-UI/api/upstream"
 	"github.com/0xJacky/Nginx-UI/internal/upstream"
 	"github.com/go-co-op/gocron/v2"
 	"github.com/uozi-tech/cosy/logger"
@@ -41,13 +40,6 @@ func executeUpstreamAvailabilityTest() {
 		return
 	}
 
-	// Check if we should skip this test due to active WebSocket connections
-	// (WebSocket connections trigger more frequent checks)
-	if hasActiveWebSocketConnections() {
-		logger.Debug("Skipping scheduled test due to active WebSocket connections")
-		return
-	}
-
 	start := time.Now()
 	logger.Debug("Starting scheduled upstream availability test for", targetCount, "targets")
 
@@ -57,11 +49,6 @@ func executeUpstreamAvailabilityTest() {
 	logger.Debug("Upstream availability test completed in", duration)
 }
 
-// hasActiveWebSocketConnections checks if there are active WebSocket connections
-func hasActiveWebSocketConnections() bool {
-	return apiUpstream.HasActiveWebSocketConnections()
-}
-
 // RestartUpstreamAvailabilityJob restarts the upstream availability job
 func RestartUpstreamAvailabilityJob() error {
 	logger.Info("Restarting upstream availability job...")

internal/site/index.go

@@ -31,6 +31,7 @@ func GetIndexedSite(path string) *SiteIndex {
 
 func init() {
 	cache.RegisterCallback(scanForSite)
+	cache.RegisterPostScanCallback(postScanSites)
 }
 
 func scanForSite(configPath string, content []byte) error {
@@ -220,8 +221,8 @@ func scanForSite(configPath string, content []byte) error {
 		siteIndex.Urls = append(siteIndex.Urls, url)
 	}
 
-	// Parse proxy targets from the configuration content
-	siteIndex.ProxyTargets = upstream.ParseProxyTargetsFromRawContent(string(content))
+	// Proxy targets will be parsed in the second pass
+	siteIndex.ProxyTargets = []ProxyTarget{}
 
 	// Only store if we found valid URLs or proxy targets
 	if len(siteIndex.Urls) > 0 || len(siteIndex.ProxyTargets) > 0 {
@@ -231,6 +232,28 @@ func scanForSite(configPath string, content []byte) error {
 	return nil
 }
 
+// postScanSites parses proxy targets for all indexed sites (second pass)
+func postScanSites() error {
+	for fileName, siteIndex := range IndexedSites {
+		if siteIndex.Content != "" {
+			proxyTargets := upstream.ParseProxyTargetsFromRawContent(siteIndex.Content)
+
+			siteIndex.ProxyTargets = make([]ProxyTarget, len(proxyTargets))
+			for i, target := range proxyTargets {
+				siteIndex.ProxyTargets[i] = ProxyTarget{
+					Host: target.Host,
+					Port: target.Port,
+					Type: target.Type,
+				}
+			}
+
+			IndexedSites[fileName] = siteIndex
+		}
+	}
+
+	return nil
+}
+
 // isValidDomain performs a basic validation of domain names
 func isValidDomain(domain string) bool {
 	// Basic validation: contains at least one dot and no spaces

internal/stream/index.go

@@ -27,6 +27,7 @@ func GetIndexedStream(path string) *StreamIndex {
 
 func init() {
 	cache.RegisterCallback(scanForStream)
+	cache.RegisterPostScanCallback(postScanStreams)
 }
 
 func scanForStream(configPath string, content []byte) error {
@@ -41,8 +42,8 @@ func scanForStream(configPath string, content []byte) error {
 		ProxyTargets: []upstream.ProxyTarget{},
 	}
 
-	// Parse proxy targets from the configuration content
-	streamIndex.ProxyTargets = upstream.ParseProxyTargetsFromRawContent(string(content))
+	// Proxy targets will be parsed in the second pass
+	streamIndex.ProxyTargets = []upstream.ProxyTarget{}
 	// Only store if we found proxy targets
 	if len(streamIndex.ProxyTargets) > 0 {
 		IndexedStreams[filepath.Base(configPath)] = &streamIndex
@@ -51,6 +52,18 @@ func scanForStream(configPath string, content []byte) error {
 	return nil
 }
 
+// postScanStreams parses proxy targets for all indexed streams (second pass)
+func postScanStreams() error {
+	for fileName, streamIndex := range IndexedStreams {
+		if streamIndex.Content != "" {
+			streamIndex.ProxyTargets = upstream.ParseProxyTargetsFromRawContent(streamIndex.Content)
+			IndexedStreams[fileName] = streamIndex
+		}
+	}
+
+	return nil
+}
+
 // isStreamConfig checks if the config path is a stream configuration
 func isStreamConfig(configPath string) bool {
 	return strings.Contains(configPath, "streams-available") ||