feat(monitoring): add enhanced monitoring features and API endpoints

- Add new API endpoints for K6 success rate and pod logs retrieval
- Enhance monitoring UI with pod details modal, YAML viewing, and improved Kafka metrics charts
- Update ClickHouse configurations with admin client and truncate operations
- Modify Kafka manager for table truncation and EPS distribution updates
- Remove outdated planning documents and binary file

BREAKING CHANGE: Updated monitoring UI structure may require frontend cache refresh

Author: rakshiths-vunet

File structure.md

@@ -7866,3 +7866,17 @@
 {"level":"error","error":"Get \"http://216.48.185.179:8086/api/system/metrics\": dial tcp 216.48.185.179:8086: connect: connection refused","node":"e2e-102-179","url":"http://216.48.185.179:8086/api/system/metrics","time":"2025-10-29T19:12:28+05:30","message":"Failed to fetch metrics from metrics API server"}
 {"level":"error","error":"Get \"http://216.48.185.179:8086/api/system/metrics\": dial tcp 216.48.185.179:8086: connect: connection refused","node":"e2e-102-179","url":"http://216.48.185.179:8086/api/system/metrics","time":"2025-10-29T19:12:31+05:30","message":"Failed to fetch metrics from metrics API server"}
 {"level":"error","error":"Get \"http://216.48.185.179:8086/api/system/metrics\": dial tcp 216.48.185.179:8086: connect: connection refused","node":"e2e-102-179","url":"http://216.48.185.179:8086/api/system/metrics","time":"2025-10-29T19:12:34+05:30","message":"Failed to fetch metrics from metrics API server"}
+{"level":"info","version":"1.0.0","time":"2025-10-30T13:54:08+05:30","message":"Starting vuDataSim Cluster Manager"}
+{"level":"info","static_dir":"./static","time":"2025-10-30T13:54:08+05:30","message":"Serving static files"}
+{"level":"info","node":"System","module":"ClickHouse","type":"info","time":"2025-10-30T13:54:08+05:30","message":"Configuration loaded successfully"}
+{"level":"info","node":"System","module":"ClickHouse","type":"info","time":"2025-10-30T13:54:08+05:30","message":"Initializing ClickHouse client connection"}
+{"level":"info","node":"System","module":"ClickHouse","type":"success","time":"2025-10-30T13:54:08+05:30","message":"ClickHouse client initialized successfully"}
+{"level":"info","node":"System","module":"ClickHouse","type":"info","time":"2025-10-30T13:54:08+05:30","message":"Initializing ClickHouse client connection"}
+{"level":"info","node":"System","module":"ClickHouse","type":"success","time":"2025-10-30T13:54:08+05:30","message":"ClickHouse client initialized successfully"}
+{"level":"info","node":"System","module":"ClickHouse","type":"success","time":"2025-10-30T13:54:08+05:30","message":"Monitoring DB client initialized successfully"}
+{"level":"info","node":"System","module":"ClickHouse","type":"success","time":"2025-10-30T13:54:08+05:30","message":"ClickHouse client initialized successfully"}
+{"level":"info","time":"2025-10-30T13:54:08+05:30","message":"ClickHouse client initialized successfully"}
+{"level":"info","port":"216.48.189.75:8086","time":"2025-10-30T13:54:08+05:30","message":"Server starting"}
+{"level":"info","url":"http://216.48.189.75:8086","time":"2025-10-30T13:54:08+05:30","message":"Open in browser"}
+{"level":"info","node":"System","module":"ClickHouse","type":"info","time":"2025-10-30T13:54:23+05:30","message":"Fetched 78 pods for namespace vsmaps"}
+{"level":"info","node":"System","module":"ClickHouse","type":"info","time":"2025-10-30T13:54:29+05:30","message":"Fetched 78 pods for namespace vsmaps"}

auto_add_nodes_plan.md

@@ -23,9 +23,10 @@ type ClickHouseConfig struct {
 
 // AppConfig holds the entire application configuration
 type AppConfig struct {
-	ClickHouse     ClickHouseConfig `yaml:"clickhouse"`
-	MonitoredNodes []string         `yaml:"monitored_nodes"`
-	MonitoringDB   ClickHouseConfig `yaml:"monitoring_db"`
+	ClickHouse      ClickHouseConfig `yaml:"clickhouse"`
+	MonitoredNodes  []string         `yaml:"monitored_nodes"`
+	MonitoringDB    ClickHouseConfig `yaml:"monitoring_db"`
+	ClickHouseAdmin ClickHouseConfig `yaml:"truncate_db"`
 }
 
 // ClickHouseClient wraps the ClickHouse connection and config
@@ -38,6 +39,12 @@ type ClickHouseClient struct {
 func NewClickHouseClient(config ClickHouseConfig) (*ClickHouseClient, error) {
 	logger.LogWithNode("System", "ClickHouse", "Initializing ClickHouse client connection", "info")
 
+	// Use higher timeout for truncate operations
+	maxExecutionTime := 300 // 5 minutes for truncate operations
+	if config.Username != "truncate_db" {
+		maxExecutionTime = 60 // 1 minute for other operations
+	}
+
 	conn, err := clickhouse.Open(&clickhouse.Options{
 		Addr: []string{fmt.Sprintf("%s:%d", config.Host, config.Port)},
 		Auth: clickhouse.Auth{
@@ -46,7 +53,7 @@ func NewClickHouseClient(config ClickHouseConfig) (*ClickHouseClient, error) {
 			Password: config.Password,
 		},
 		Settings: clickhouse.Settings{
-			"max_execution_time": 60,
+			"max_execution_time": maxExecutionTime,
 		},
 		Compression: &clickhouse.Compression{
 			Method: clickhouse.CompressionLZ4,
@@ -87,6 +94,8 @@ var clickHouseClient *ClickHouseClient
 var clickHouseConfig ClickHouseConfig
 var monitoringDBClient *ClickHouseClient
 var monitoringDBConfig ClickHouseConfig
+var clickHouseAdminClient *ClickHouseClient
+var clickHouseAdminConfig ClickHouseConfig
 var monitoredNodes []string
 
 // LoadConfig loads configuration from YAML file
@@ -104,6 +113,7 @@ func LoadConfig(configPath string) error {
 
 	clickHouseConfig = config.ClickHouse
 	monitoringDBConfig = config.MonitoringDB
+	clickHouseAdminConfig = config.ClickHouseAdmin
 	monitoredNodes = config.MonitoredNodes
 
 	logger.LogWithNode("System", "ClickHouse", "Configuration loaded successfully", "info")
@@ -135,6 +145,17 @@ func InitClickHouse(configPath string) error {
 		}
 	}
 
+	// Initialize admin client if configured
+	if clickHouseAdminConfig.Host != "" {
+		adminClient, err := NewClickHouseClient(clickHouseAdminConfig)
+		if err != nil {
+			logger.LogWarning("System", "ClickHouse", fmt.Sprintf("Failed to initialize ClickHouse admin client: %v", err))
+		} else {
+			clickHouseAdminClient = adminClient
+			logger.LogSuccess("System", "ClickHouse", "ClickHouse admin client initialized successfully")
+		}
+	}
+
 	logger.LogSuccess("System", "ClickHouse", "ClickHouse client initialized successfully")
 	return nil
 }
@@ -171,3 +192,13 @@ func GetMonitoredNodes() []string {
 func GetClickHouseClient() *ClickHouseClient {
 	return clickHouseClient
 }
+
+// GetClickHouseAdminClient returns the global ClickHouse admin client
+func GetClickHouseAdminClient() *ClickHouseClient {
+	return clickHouseAdminClient
+}
+
+// GetMonitoringDBClient returns the global monitoring DB client
+func GetMonitoringDBClient() *ClickHouseClient {
+	return monitoringDBClient
+}