服务端媒体源导入/保存/客户端输出链路修复:支持 snake/camel、subcategories/sources,默认客户端可见,保存后发布兼容 media-types.json。
build-winui / winui (push) Has been cancelled
build-winui / winui (push) Has been cancelled
新增数据库同步 Job API、持久化状态、实时输出、最新任务恢复,以及系统日志聚合接口。 管理端优化:日志中心、运维实时状态框、同步输出自动滚动、仪表盘“输出”列、真实延迟空态、本地 favicon/avatar。 新增 server/unified-management/assets/favicon.ico 和 developer-avatar.png,并接好 /favicon.ico、/admin/favicon.ico、/setup/favicon.ico、/assets/*。 WinUI 随机放映室卡片优先显示子接口原始 Description。 Inno 安装器输出框改为选区末尾 + SendMessage 滚动到底部。
This commit is contained in:
@@ -21,31 +21,80 @@ func (s *Store) CopyRemoteToSQLite() (string, error) {
|
||||
}
|
||||
|
||||
func (s *Store) ImportSQLiteToRemote() (SyncResult, error) {
|
||||
if !s.trySyncLock() {
|
||||
return SyncResult{Direction: "sqlite_to_remote", Status: "running", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"database sync is already running"}}, errors.New("database sync is already running")
|
||||
}
|
||||
defer s.syncMu.Unlock()
|
||||
s.mu.RLock()
|
||||
remote := s.remoteDB
|
||||
remoteDialect := s.remoteDialect
|
||||
local := s.localDB
|
||||
localDialect := s.localDialect
|
||||
s.mu.RUnlock()
|
||||
if remote == nil {
|
||||
result := SyncResult{Direction: "sqlite_to_remote", Status: "skipped", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"remote database is not configured"}}
|
||||
s.setSyncStatus(result, nil)
|
||||
return result, nil
|
||||
}
|
||||
result, err := copyAllTables(local, localDialect, remote, remoteDialect, "sqlite_to_remote")
|
||||
s.setSyncStatus(result, err)
|
||||
return result, err
|
||||
return s.RunDatabaseSync("sqlite_to_remote")
|
||||
}
|
||||
|
||||
func (s *Store) SyncNow() (SyncResult, error) {
|
||||
return s.RunDatabaseSync("remote_to_sqlite")
|
||||
}
|
||||
|
||||
func (s *Store) QueueDatabaseSync(direction string) (DatabaseSyncJob, error) {
|
||||
normalized, err := normalizeSyncDirection(direction)
|
||||
if err != nil {
|
||||
return DatabaseSyncJob{}, err
|
||||
}
|
||||
if !s.trySyncLock() {
|
||||
return SyncResult{Direction: "remote_to_sqlite", Status: "running", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"database sync is already running"}}, errors.New("database sync is already running")
|
||||
job, insertErr := s.insertDatabaseSyncJob(normalized, "skipped", []string{"同步任务未启动:已有数据库同步正在执行。"}, []string{"database sync is already running"}, nil, nil, Now())
|
||||
if insertErr != nil {
|
||||
return DatabaseSyncJob{}, insertErr
|
||||
}
|
||||
return job, errors.New("database sync is already running")
|
||||
}
|
||||
job, err := s.insertDatabaseSyncJob(normalized, "running", []string{"同步任务已创建,正在准备数据库连接。"}, nil, nil, map[string]int{}, "")
|
||||
if err != nil {
|
||||
s.syncMu.Unlock()
|
||||
return DatabaseSyncJob{}, err
|
||||
}
|
||||
s.setCurrentSyncJob(&job)
|
||||
go func(jobID int64, jobDirection string) {
|
||||
defer s.syncMu.Unlock()
|
||||
_, _ = s.runDatabaseSyncLocked(jobID, jobDirection)
|
||||
}(job.ID, normalized)
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func (s *Store) RunDatabaseSync(direction string) (SyncResult, error) {
|
||||
normalized, err := normalizeSyncDirection(direction)
|
||||
if err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
if !s.trySyncLock() {
|
||||
result := SyncResult{
|
||||
Direction: normalized,
|
||||
Status: "skipped",
|
||||
Skipped: true,
|
||||
StartedAt: Now(),
|
||||
FinishedAt: Now(),
|
||||
Tables: map[string]int{},
|
||||
Warnings: []string{"database sync is already running"},
|
||||
Output: []string{"同步任务未启动:已有数据库同步正在执行。"},
|
||||
}
|
||||
_, _ = s.insertDatabaseSyncJob(normalized, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
|
||||
s.setSyncStatus(result, errors.New("database sync is already running"))
|
||||
return result, errors.New("database sync is already running")
|
||||
}
|
||||
defer s.syncMu.Unlock()
|
||||
job, err := s.insertDatabaseSyncJob(normalized, "running", []string{"同步任务已创建,正在准备数据库连接。"}, nil, nil, map[string]int{}, "")
|
||||
if err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
s.setCurrentSyncJob(&job)
|
||||
return s.runDatabaseSyncLocked(job.ID, normalized)
|
||||
}
|
||||
|
||||
func (s *Store) runDatabaseSyncLocked(jobID int64, direction string) (SyncResult, error) {
|
||||
started := Now()
|
||||
if job, err := s.GetDatabaseSyncJob(jobID); err == nil && job.StartedAt != "" {
|
||||
started = job.StartedAt
|
||||
}
|
||||
result := SyncResult{
|
||||
Direction: direction,
|
||||
Status: "completed",
|
||||
StartedAt: started,
|
||||
Tables: map[string]int{},
|
||||
Output: []string{"同步开始:" + directionLabel(direction)},
|
||||
}
|
||||
_ = s.updateDatabaseSyncJob(jobID, "running", result.Output, result.Warnings, result.Errors, result.Tables, "")
|
||||
s.mu.RLock()
|
||||
remote := s.remoteDB
|
||||
remoteDialect := s.remoteDialect
|
||||
@@ -53,18 +102,43 @@ func (s *Store) SyncNow() (SyncResult, error) {
|
||||
localDialect := s.localDialect
|
||||
s.mu.RUnlock()
|
||||
if remote == nil {
|
||||
result := SyncResult{Direction: "remote_to_sqlite", Status: "skipped", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"remote database is not configured"}}
|
||||
result.Status = "skipped"
|
||||
result.Skipped = true
|
||||
result.Warnings = append(result.Warnings, "remote database is not configured")
|
||||
result.Output = append(result.Output, "远程 MySQL 未配置或不可用,未执行数据复制。")
|
||||
result.FinishedAt = Now()
|
||||
_ = s.updateDatabaseSyncJob(jobID, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
|
||||
s.setSyncStatus(result, nil)
|
||||
return result, nil
|
||||
}
|
||||
result, err := copyAllTables(remote, remoteDialect, local, localDialect, "remote_to_sqlite")
|
||||
s.setSyncStatus(result, err)
|
||||
return result, err
|
||||
src, srcDialect, dst, dstDialect := local, localDialect, remote, remoteDialect
|
||||
if direction == "remote_to_sqlite" {
|
||||
src, srcDialect, dst, dstDialect = remote, remoteDialect, local, localDialect
|
||||
}
|
||||
err := copyAllTablesWithProgress(src, srcDialect, dst, dstDialect, direction, func(table string, count int) {
|
||||
result.Tables[table] = count
|
||||
result.Output = append(result.Output, fmt.Sprintf("%s:%d 条", table, count))
|
||||
_ = s.updateDatabaseSyncJob(jobID, "running", result.Output, result.Warnings, result.Errors, result.Tables, "")
|
||||
})
|
||||
result.FinishedAt = Now()
|
||||
if err != nil {
|
||||
result.Status = "failed"
|
||||
result.Errors = append(result.Errors, err.Error())
|
||||
result.Output = append(result.Output, "同步失败:"+err.Error())
|
||||
_ = s.updateDatabaseSyncJob(jobID, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
|
||||
s.setSyncStatus(result, err)
|
||||
return result, err
|
||||
}
|
||||
result.Output = append(result.Output, "同步完成:"+directionLabel(direction))
|
||||
_ = s.updateDatabaseSyncJob(jobID, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
|
||||
s.setSyncStatus(result, nil)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) setSyncStatus(result SyncResult, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.status.CurrentSyncJob = nil
|
||||
if err != nil {
|
||||
s.status.LastSyncAt = result.FinishedAt
|
||||
s.status.LastSyncError = err.Error()
|
||||
@@ -78,6 +152,33 @@ func (s *Store) trySyncLock() bool {
|
||||
return s.syncMu.TryLock()
|
||||
}
|
||||
|
||||
func (s *Store) setCurrentSyncJob(job *DatabaseSyncJob) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.status.CurrentSyncJob = job
|
||||
}
|
||||
|
||||
func normalizeSyncDirection(value string) (string, error) {
|
||||
switch strings.ToLower(strings.TrimSpace(value)) {
|
||||
case "sqlite_to_remote", "import", "sqlite_to_mysql":
|
||||
return "sqlite_to_remote", nil
|
||||
case "remote_to_sqlite", "sync", "mysql_to_sqlite":
|
||||
return "remote_to_sqlite", nil
|
||||
default:
|
||||
return "", errors.New("unsupported database sync direction")
|
||||
}
|
||||
}
|
||||
|
||||
func directionLabel(direction string) string {
|
||||
if direction == "sqlite_to_remote" {
|
||||
return "SQLite -> MySQL"
|
||||
}
|
||||
if direction == "remote_to_sqlite" {
|
||||
return "MySQL -> SQLite"
|
||||
}
|
||||
return direction
|
||||
}
|
||||
|
||||
type tableSpec struct {
|
||||
Name string
|
||||
Columns []string
|
||||
@@ -100,6 +201,7 @@ var syncTables = []tableSpec{
|
||||
{"source_endpoints", []string{"id", "category_id", "category_name", "source_id", "name", "description", "method", "api_url", "url_template", "thumbnail_url", "proxy_mode", "timeout_ms", "retry_count", "cache_seconds", "check_interval_sec", "enabled", "client_visible", "supported_formats", "last_status", "last_latency_ms", "last_checked_at", "last_error", "consecutive_failure", "created_at", "updated_at"}, []string{"id"}},
|
||||
{"endpoint_health_checks", []string{"id", "source_db_id", "status", "latency_ms", "error", "checked_at"}, []string{"id"}},
|
||||
{"endpoint_call_logs", []string{"id", "source_id", "status", "latency_ms", "error", "client", "created_at"}, []string{"id"}},
|
||||
{"database_sync_jobs", []string{"id", "direction", "status", "message", "tables_json", "started_at", "finished_at"}, []string{"id"}},
|
||||
{"system_settings", []string{"key", "value", "updated_at"}, []string{"key"}},
|
||||
{"audit_logs", []string{"id", "actor", "type", "target", "message", "ip", "user_agent", "created_at"}, []string{"id"}},
|
||||
{"legacy_json_revisions", []string{"id", "name", "raw", "note", "created_by", "created_at"}, []string{"id"}},
|
||||
@@ -109,19 +211,31 @@ var syncTables = []tableSpec{
|
||||
|
||||
func copyAllTables(src *sql.DB, srcDialect dialect, dst *sql.DB, dstDialect dialect, direction string) (SyncResult, error) {
|
||||
result := SyncResult{Direction: direction, Status: "completed", Tables: map[string]int{}, FinishedAt: Now()}
|
||||
for _, table := range syncTables {
|
||||
count, err := copyTable(src, srcDialect, dst, dstDialect, table)
|
||||
if err != nil {
|
||||
result.Status = "failed"
|
||||
result.FinishedAt = Now()
|
||||
return result, err
|
||||
}
|
||||
result.Tables[table.Name] = count
|
||||
err := copyAllTablesWithProgress(src, srcDialect, dst, dstDialect, direction, func(table string, count int) {
|
||||
result.Tables[table] = count
|
||||
})
|
||||
if err != nil {
|
||||
result.Status = "failed"
|
||||
result.FinishedAt = Now()
|
||||
return result, err
|
||||
}
|
||||
result.FinishedAt = Now()
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func copyAllTablesWithProgress(src *sql.DB, srcDialect dialect, dst *sql.DB, dstDialect dialect, direction string, progress func(table string, count int)) error {
|
||||
for _, table := range syncTables {
|
||||
count, err := copyTable(src, srcDialect, dst, dstDialect, table)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if progress != nil {
|
||||
progress(table.Name, count)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyTable(src *sql.DB, srcDialect dialect, dst *sql.DB, dstDialect dialect, spec tableSpec) (int, error) {
|
||||
rows, err := src.Query(srcDialect.rebind("SELECT " + srcDialect.columnList(spec.Columns) + " FROM " + spec.Name))
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type syncJobMessage struct {
|
||||
Output []string `json:"output"`
|
||||
Warnings []string `json:"warnings"`
|
||||
Errors []string `json:"errors"`
|
||||
}
|
||||
|
||||
type syncJobScanner interface {
|
||||
Scan(dest ...any) error
|
||||
}
|
||||
|
||||
func (s *Store) insertDatabaseSyncJob(direction, status string, output, warnings, jobErrors []string, tables map[string]int, finishedAt string) (DatabaseSyncJob, error) {
|
||||
startedAt := Now()
|
||||
message, tablesJSON := encodeDatabaseSyncJob(output, warnings, jobErrors, tables)
|
||||
id, err := s.insertID(`INSERT INTO database_sync_jobs (direction, status, message, tables_json, started_at, finished_at) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
direction, status, message, tablesJSON, startedAt, finishedAt)
|
||||
if err != nil {
|
||||
return DatabaseSyncJob{}, err
|
||||
}
|
||||
return s.GetDatabaseSyncJob(id)
|
||||
}
|
||||
|
||||
func (s *Store) updateDatabaseSyncJob(id int64, status string, output, warnings, jobErrors []string, tables map[string]int, finishedAt string) error {
|
||||
message, tablesJSON := encodeDatabaseSyncJob(output, warnings, jobErrors, tables)
|
||||
if _, err := s.exec(`UPDATE database_sync_jobs SET status = ?, message = ?, tables_json = ?, finished_at = ? WHERE id = ?`,
|
||||
status, message, tablesJSON, finishedAt, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if job, err := s.GetDatabaseSyncJob(id); err == nil {
|
||||
s.mu.Lock()
|
||||
if s.status.CurrentSyncJob != nil && s.status.CurrentSyncJob.ID == id {
|
||||
s.status.CurrentSyncJob = &job
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetDatabaseSyncJob(id int64) (DatabaseSyncJob, error) {
|
||||
job, err := scanDatabaseSyncJob(s.queryRow(`SELECT id, direction, status, message, tables_json, started_at, finished_at FROM database_sync_jobs WHERE id = ?`, id))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return DatabaseSyncJob{}, errors.New("database sync job not found")
|
||||
}
|
||||
return job, err
|
||||
}
|
||||
|
||||
func (s *Store) LatestDatabaseSyncJob() (DatabaseSyncJob, error) {
|
||||
job, err := scanDatabaseSyncJob(s.queryRow(`SELECT id, direction, status, message, tables_json, started_at, finished_at FROM database_sync_jobs ORDER BY id DESC LIMIT 1`))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return DatabaseSyncJob{}, errors.New("database sync job not found")
|
||||
}
|
||||
return job, err
|
||||
}
|
||||
|
||||
func (s *Store) restoreDatabaseSyncStatus() {
|
||||
job, err := s.LatestDatabaseSyncJob()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if job.Status == "running" {
|
||||
s.status.CurrentSyncJob = &job
|
||||
} else {
|
||||
s.status.CurrentSyncJob = nil
|
||||
}
|
||||
if job.FinishedAt != "" {
|
||||
s.status.LastSyncAt = job.FinishedAt
|
||||
} else {
|
||||
s.status.LastSyncAt = job.StartedAt
|
||||
}
|
||||
if job.Status == "failed" {
|
||||
s.status.LastSyncError = strings.Join(job.Errors, "; ")
|
||||
} else {
|
||||
s.status.LastSyncError = ""
|
||||
}
|
||||
}
|
||||
|
||||
func scanDatabaseSyncJob(scanner syncJobScanner) (DatabaseSyncJob, error) {
|
||||
var job DatabaseSyncJob
|
||||
var message, tablesJSON string
|
||||
if err := scanner.Scan(&job.ID, &job.Direction, &job.Status, &message, &tablesJSON, &job.StartedAt, &job.FinishedAt); err != nil {
|
||||
return DatabaseSyncJob{}, err
|
||||
}
|
||||
decodeDatabaseSyncJob(message, tablesJSON, &job)
|
||||
return job, nil
|
||||
}
|
||||
|
||||
func encodeDatabaseSyncJob(output, warnings, jobErrors []string, tables map[string]int) (string, string) {
|
||||
if output == nil {
|
||||
output = []string{}
|
||||
}
|
||||
if warnings == nil {
|
||||
warnings = []string{}
|
||||
}
|
||||
if jobErrors == nil {
|
||||
jobErrors = []string{}
|
||||
}
|
||||
if tables == nil {
|
||||
tables = map[string]int{}
|
||||
}
|
||||
message, _ := json.Marshal(syncJobMessage{Output: output, Warnings: warnings, Errors: jobErrors})
|
||||
tableData, _ := json.Marshal(tables)
|
||||
return string(message), string(tableData)
|
||||
}
|
||||
|
||||
func decodeDatabaseSyncJob(message, tablesJSON string, job *DatabaseSyncJob) {
|
||||
job.Output = []string{}
|
||||
job.Warnings = []string{}
|
||||
job.Errors = []string{}
|
||||
job.Tables = map[string]int{}
|
||||
var payload syncJobMessage
|
||||
if strings.TrimSpace(message) != "" && json.Unmarshal([]byte(message), &payload) == nil {
|
||||
job.Output = payload.Output
|
||||
job.Warnings = payload.Warnings
|
||||
job.Errors = payload.Errors
|
||||
} else if strings.TrimSpace(message) != "" {
|
||||
job.Output = []string{message}
|
||||
}
|
||||
_ = json.Unmarshal([]byte(tablesJSON), &job.Tables)
|
||||
if job.Output == nil {
|
||||
job.Output = []string{}
|
||||
}
|
||||
if job.Warnings == nil {
|
||||
job.Warnings = []string{}
|
||||
}
|
||||
if job.Errors == nil {
|
||||
job.Errors = []string{}
|
||||
}
|
||||
if job.Tables == nil {
|
||||
job.Tables = map[string]int{}
|
||||
}
|
||||
}
|
||||
@@ -20,17 +20,20 @@ type adminRow struct {
|
||||
}
|
||||
|
||||
type DatabaseStatus struct {
|
||||
ActiveProvider string `json:"activeProvider"`
|
||||
ConfigProvider string `json:"configProvider"`
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
SQLiteReady bool `json:"sqliteReady"`
|
||||
RemoteReady bool `json:"remoteReady"`
|
||||
FailoverActive bool `json:"failoverActive"`
|
||||
LastError string `json:"lastError"`
|
||||
LastFailoverAt string `json:"lastFailoverAt"`
|
||||
LastRecoveredAt string `json:"lastRecoveredAt"`
|
||||
LastSyncAt string `json:"lastSyncAt"`
|
||||
LastSyncError string `json:"lastSyncError"`
|
||||
ActiveProvider string `json:"activeProvider"`
|
||||
ConfigProvider string `json:"configProvider"`
|
||||
SchemaVersion string `json:"schemaVersion"`
|
||||
SQLiteReady bool `json:"sqliteReady"`
|
||||
RemoteReady bool `json:"remoteReady"`
|
||||
FailoverActive bool `json:"failoverActive"`
|
||||
LastError string `json:"lastError"`
|
||||
LastFailoverAt string `json:"lastFailoverAt"`
|
||||
LastRecoveredAt string `json:"lastRecoveredAt"`
|
||||
LastSyncAt string `json:"lastSyncAt"`
|
||||
LastSyncError string `json:"lastSyncError"`
|
||||
LastHealthCheckedAt string `json:"lastHealthCheckedAt"`
|
||||
LastHealthStatus string `json:"lastHealthStatus"`
|
||||
CurrentSyncJob *DatabaseSyncJob `json:"currentSyncJob,omitempty"`
|
||||
}
|
||||
|
||||
type SyncResult struct {
|
||||
@@ -38,10 +41,25 @@ type SyncResult struct {
|
||||
Status string `json:"status"`
|
||||
Skipped bool `json:"skipped"`
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Output []string `json:"output,omitempty"`
|
||||
StartedAt string `json:"startedAt,omitempty"`
|
||||
Tables map[string]int `json:"tables"`
|
||||
FinishedAt string `json:"finishedAt"`
|
||||
}
|
||||
|
||||
type DatabaseSyncJob struct {
|
||||
ID int64 `json:"id"`
|
||||
Direction string `json:"direction"`
|
||||
Status string `json:"status"`
|
||||
Output []string `json:"output"`
|
||||
Warnings []string `json:"warnings"`
|
||||
Errors []string `json:"errors"`
|
||||
Tables map[string]int `json:"tables"`
|
||||
StartedAt string `json:"startedAt"`
|
||||
FinishedAt string `json:"finishedAt"`
|
||||
}
|
||||
|
||||
type AdminUser struct {
|
||||
ID int64 `json:"id"`
|
||||
Username string `json:"username"`
|
||||
@@ -246,6 +264,8 @@ type Source struct {
|
||||
ConsecutiveFailure int `json:"consecutiveFailure"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
UpdatedAt string `json:"updatedAt"`
|
||||
EnabledSet bool `json:"-"`
|
||||
ClientVisibleSet bool `json:"-"`
|
||||
}
|
||||
|
||||
type SourceCheck struct {
|
||||
@@ -301,3 +321,28 @@ type LegacyJsonRevision struct {
|
||||
CreatedBy string `json:"createdBy"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
}
|
||||
|
||||
type SystemLogItem struct {
|
||||
ID int64 `json:"id"`
|
||||
Category string `json:"category"`
|
||||
Type string `json:"type"`
|
||||
Target string `json:"target"`
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Detail string `json:"detail"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
}
|
||||
|
||||
type SystemLogPage struct {
|
||||
Items []SystemLogItem `json:"items"`
|
||||
Total int `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PerPage int `json:"perPage"`
|
||||
}
|
||||
|
||||
type SystemLogFilters struct {
|
||||
Page int
|
||||
PerPage int
|
||||
Category string
|
||||
Query string
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@ func (s *Store) UpsertSource(item Source) (Source, error) {
|
||||
if item.SourceID == "" {
|
||||
item.SourceID = item.CategoryID + "-" + item.Name
|
||||
}
|
||||
existing, existingErr := s.GetSourceBySourceID(item.SourceID)
|
||||
hasExisting := existingErr == nil
|
||||
if item.Method == "" {
|
||||
item.Method = "GET"
|
||||
}
|
||||
@@ -33,10 +35,30 @@ func (s *Store) UpsertSource(item Source) (Source, error) {
|
||||
item.CheckIntervalSec = item.CacheSeconds
|
||||
}
|
||||
if item.SupportedFormats == "" {
|
||||
item.SupportedFormats = "[]"
|
||||
if hasExisting && existing.SupportedFormats != "" {
|
||||
item.SupportedFormats = existing.SupportedFormats
|
||||
} else {
|
||||
item.SupportedFormats = "[]"
|
||||
}
|
||||
}
|
||||
if item.LastStatus == "" {
|
||||
item.LastStatus = "unknown"
|
||||
if hasExisting && existing.LastStatus != "" {
|
||||
item.LastStatus = existing.LastStatus
|
||||
} else {
|
||||
item.LastStatus = "unknown"
|
||||
}
|
||||
}
|
||||
if item.LastLatencyMS == 0 && hasExisting {
|
||||
item.LastLatencyMS = existing.LastLatencyMS
|
||||
}
|
||||
if item.LastCheckedAt == "" && hasExisting {
|
||||
item.LastCheckedAt = existing.LastCheckedAt
|
||||
}
|
||||
if item.LastError == "" && hasExisting {
|
||||
item.LastError = existing.LastError
|
||||
}
|
||||
if item.ConsecutiveFailure == 0 && hasExisting {
|
||||
item.ConsecutiveFailure = existing.ConsecutiveFailure
|
||||
}
|
||||
if item.CategoryID == "" {
|
||||
item.CategoryID = "custom"
|
||||
@@ -44,6 +66,21 @@ func (s *Store) UpsertSource(item Source) (Source, error) {
|
||||
if item.CategoryName == "" {
|
||||
item.CategoryName = item.CategoryID
|
||||
}
|
||||
if !item.EnabledSet {
|
||||
item.Enabled = true
|
||||
if hasExisting {
|
||||
item.Enabled = existing.Enabled
|
||||
}
|
||||
}
|
||||
if !item.ClientVisibleSet {
|
||||
item.ClientVisible = true
|
||||
if hasExisting {
|
||||
item.ClientVisible = existing.ClientVisible
|
||||
}
|
||||
}
|
||||
if item.CreatedAt == "" && hasExisting {
|
||||
item.CreatedAt = existing.CreatedAt
|
||||
}
|
||||
_, _ = s.exec(`INSERT INTO source_categories (category_id, name, enabled, ui_config, created_at, updated_at)
|
||||
VALUES (?, ?, 1, '{}', ?, ?)
|
||||
ON CONFLICT (category_id) DO UPDATE SET name = excluded.name, updated_at = excluded.updated_at`,
|
||||
@@ -97,6 +134,12 @@ func (s *Store) CountSources() (int, error) {
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (s *Store) CountClientVisibleSources() (int, error) {
|
||||
var count int
|
||||
err := s.queryRow(`SELECT COUNT(*) FROM source_endpoints WHERE enabled = 1 AND client_visible = 1`).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
func (s *Store) DeleteSource(sourceID string) error {
|
||||
_, err := s.exec(`DELETE FROM source_endpoints WHERE source_id = ?`, sourceID)
|
||||
return err
|
||||
|
||||
@@ -69,6 +69,7 @@ func Open(cfg *config.Config) (*Store, error) {
|
||||
SchemaVersion: CurrentSchemaVersion,
|
||||
SQLiteReady: true,
|
||||
LastRecoveredAt: Now(),
|
||||
LastHealthStatus: "not_configured",
|
||||
},
|
||||
}
|
||||
if err := store.migrate(local, localDialect); err != nil {
|
||||
@@ -86,6 +87,7 @@ func Open(cfg *config.Config) (*Store, error) {
|
||||
store.markFailover(err)
|
||||
}
|
||||
}
|
||||
store.restoreDatabaseSyncStatus()
|
||||
go store.maintain()
|
||||
return store, nil
|
||||
}
|
||||
@@ -171,14 +173,17 @@ func (s *Store) ReconfigureDatabase(cfg *config.Config) error {
|
||||
s.status.RemoteReady = remote != nil
|
||||
s.status.LastError = ""
|
||||
s.status.FailoverActive = false
|
||||
s.status.LastHealthCheckedAt = Now()
|
||||
if remote != nil {
|
||||
s.db = remote
|
||||
s.dialect = remoteDialect
|
||||
s.status.ActiveProvider = "mysql"
|
||||
s.status.LastHealthStatus = "ok"
|
||||
} else {
|
||||
s.db = local
|
||||
s.dialect = localDialect
|
||||
s.status.ActiveProvider = "sqlite"
|
||||
s.status.LastHealthStatus = "not_configured"
|
||||
}
|
||||
s.status.LastRecoveredAt = Now()
|
||||
s.mu.Unlock()
|
||||
@@ -188,6 +193,7 @@ func (s *Store) ReconfigureDatabase(cfg *config.Config) error {
|
||||
if oldLocal != nil && oldLocal != local {
|
||||
_ = oldLocal.Close()
|
||||
}
|
||||
s.restoreDatabaseSyncStatus()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -237,7 +243,11 @@ func (s *Store) insertID(query string, args ...any) (int64, error) {
|
||||
}
|
||||
|
||||
func (s *Store) maintain() {
|
||||
ticker := time.NewTicker(time.Duration(s.cfg.Database.HealthIntervalSec) * time.Second)
|
||||
interval := s.cfg.Database.HealthIntervalSec
|
||||
if interval <= 0 {
|
||||
interval = 60
|
||||
}
|
||||
ticker := time.NewTicker(time.Duration(interval) * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
@@ -276,12 +286,18 @@ func (s *Store) openRemote() error {
|
||||
s.status.FailoverActive = false
|
||||
s.status.LastError = ""
|
||||
s.status.LastRecoveredAt = Now()
|
||||
s.status.LastHealthCheckedAt = Now()
|
||||
s.status.LastHealthStatus = "ok"
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) checkRemote() {
|
||||
if !strings.EqualFold(s.cfg.Database.Provider, "mysql") {
|
||||
s.mu.Lock()
|
||||
s.status.LastHealthCheckedAt = Now()
|
||||
s.status.LastHealthStatus = "not_configured"
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.mu.RLock()
|
||||
@@ -311,6 +327,8 @@ func (s *Store) checkRemote() {
|
||||
s.status.FailoverActive = false
|
||||
s.status.LastError = ""
|
||||
s.status.LastRecoveredAt = Now()
|
||||
s.status.LastHealthCheckedAt = Now()
|
||||
s.status.LastHealthStatus = "ok"
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
@@ -329,4 +347,6 @@ func (s *Store) markFailover(err error) {
|
||||
s.status.FailoverActive = !strings.EqualFold(s.cfg.Database.Provider, "sqlite")
|
||||
s.status.LastError = err.Error()
|
||||
s.status.LastFailoverAt = Now()
|
||||
s.status.LastHealthCheckedAt = Now()
|
||||
s.status.LastHealthStatus = "error"
|
||||
}
|
||||
|
||||
@@ -391,3 +391,47 @@ func TestDashboardOverviewKeepsChecksForDeletedSources(t *testing.T) {
|
||||
t.Fatalf("deleted source check should have fallback sourceId/name: %#v", checks[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDatabaseSyncJobPersistsLatestStatusAndOutput(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
path := filepath.Join(root, "unified.sqlite")
|
||||
store, err := Open(&config.Config{
|
||||
StorageDir: root,
|
||||
Database: config.DatabaseConfig{
|
||||
Provider: "sqlite",
|
||||
SQLitePath: path,
|
||||
FailoverEnabled: true,
|
||||
HealthIntervalSec: 3600,
|
||||
MaxOpenConns: 1,
|
||||
MaxIdleConns: 1,
|
||||
ConnMaxLifetimeSeconds: 60,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
result, err := store.ImportSQLiteToRemote()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !result.Skipped || result.Status != "skipped" {
|
||||
t.Fatalf("expected missing remote sync to be skipped, got %#v", result)
|
||||
}
|
||||
latest, err := store.LatestDatabaseSyncJob()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if latest.Direction != "sqlite_to_remote" || latest.Status != "skipped" {
|
||||
t.Fatalf("unexpected latest sync job: %#v", latest)
|
||||
}
|
||||
if len(latest.Output) == 0 || latest.Tables == nil {
|
||||
t.Fatalf("latest job should include output and tables: %#v", latest)
|
||||
}
|
||||
store.restoreDatabaseSyncStatus()
|
||||
status := store.Status()
|
||||
if status.LastSyncAt == "" || status.LastSyncError != "" {
|
||||
t.Fatalf("sync status was not restored from latest job: %#v", status)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (s *Store) ListSystemLogsPage(filters SystemLogFilters) (SystemLogPage, error) {
|
||||
page := filters.Page
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
perPage := filters.PerPage
|
||||
if perPage <= 0 {
|
||||
perPage = 35
|
||||
}
|
||||
if perPage > 100 {
|
||||
perPage = 100
|
||||
}
|
||||
items, err := s.collectSystemLogs(filters)
|
||||
if err != nil {
|
||||
return SystemLogPage{}, err
|
||||
}
|
||||
sort.SliceStable(items, func(i, j int) bool {
|
||||
if items[i].CreatedAt == items[j].CreatedAt {
|
||||
return items[i].ID > items[j].ID
|
||||
}
|
||||
return items[i].CreatedAt > items[j].CreatedAt
|
||||
})
|
||||
total := len(items)
|
||||
start := (page - 1) * perPage
|
||||
if start > total {
|
||||
start = total
|
||||
}
|
||||
end := start + perPage
|
||||
if end > total {
|
||||
end = total
|
||||
}
|
||||
return SystemLogPage{Items: items[start:end], Total: total, Page: page, PerPage: perPage}, nil
|
||||
}
|
||||
|
||||
func (s *Store) collectSystemLogs(filters SystemLogFilters) ([]SystemLogItem, error) {
|
||||
items := []SystemLogItem{}
|
||||
appendItems := func(category string, next []SystemLogItem, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range next {
|
||||
if matchesSystemLog(filters, item, category) {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
logs, err := s.operationLogs()
|
||||
if err := appendItems("operation", logs, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err = s.healthLogs()
|
||||
if err := appendItems("health", logs, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err = s.clientCallLogs()
|
||||
if err := appendItems("client", logs, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err = s.databaseSyncLogs()
|
||||
if err := appendItems("database_sync", logs, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logs, err = s.legacySyncLogs()
|
||||
if err := appendItems("legacy_sync", logs, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func (s *Store) operationLogs() ([]SystemLogItem, error) {
|
||||
rows, err := s.query(`SELECT id, actor, type, target, message, ip, user_agent, created_at FROM audit_logs ORDER BY id DESC LIMIT 500`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []SystemLogItem{}
|
||||
for rows.Next() {
|
||||
var item AuditLog
|
||||
if err := rows.Scan(&item.ID, &item.Actor, &item.Type, &item.Target, &item.Message, &item.IP, &item.UserAgent, &item.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, SystemLogItem{
|
||||
ID: item.ID,
|
||||
Category: "operation",
|
||||
Type: item.Type,
|
||||
Target: item.Target,
|
||||
Status: firstNonEmpty(item.Actor, "system"),
|
||||
Message: item.Message,
|
||||
Detail: strings.TrimSpace(item.IP + " " + item.UserAgent),
|
||||
CreatedAt: item.CreatedAt,
|
||||
})
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) healthLogs() ([]SystemLogItem, error) {
|
||||
rows, err := s.query(`SELECT h.id, h.source_db_id, COALESCE(e.source_id, ''), COALESCE(e.name, ''), h.status, h.latency_ms, h.error, h.checked_at
|
||||
FROM endpoint_health_checks h LEFT JOIN source_endpoints e ON e.id = h.source_db_id
|
||||
ORDER BY h.id DESC LIMIT 500`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []SystemLogItem{}
|
||||
for rows.Next() {
|
||||
var id, sourceDBID int64
|
||||
var sourceID, name, status, message, checkedAt string
|
||||
var latency int
|
||||
if err := rows.Scan(&id, &sourceDBID, &sourceID, &name, &status, &latency, &message, &checkedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
target := firstNonEmpty(sourceID, fmt.Sprintf("source#%d", sourceDBID))
|
||||
items = append(items, SystemLogItem{
|
||||
ID: id,
|
||||
Category: "health",
|
||||
Type: "endpoint.health",
|
||||
Target: target,
|
||||
Status: status,
|
||||
Message: firstNonEmpty(name, target),
|
||||
Detail: fmt.Sprintf("%dms %s", latency, message),
|
||||
CreatedAt: checkedAt,
|
||||
})
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) clientCallLogs() ([]SystemLogItem, error) {
|
||||
rows, err := s.query(`SELECT id, source_id, status, latency_ms, error, client, created_at FROM endpoint_call_logs ORDER BY id DESC LIMIT 500`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []SystemLogItem{}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var sourceID, status, message, client, createdAt string
|
||||
var latency int
|
||||
if err := rows.Scan(&id, &sourceID, &status, &latency, &message, &client, &createdAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, SystemLogItem{
|
||||
ID: id,
|
||||
Category: "client",
|
||||
Type: "endpoint.call",
|
||||
Target: sourceID,
|
||||
Status: status,
|
||||
Message: message,
|
||||
Detail: fmt.Sprintf("%dms %s", latency, client),
|
||||
CreatedAt: createdAt,
|
||||
})
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) databaseSyncLogs() ([]SystemLogItem, error) {
|
||||
rows, err := s.query(`SELECT id, direction, status, message, tables_json, started_at, finished_at FROM database_sync_jobs ORDER BY id DESC LIMIT 500`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []SystemLogItem{}
|
||||
for rows.Next() {
|
||||
job, err := scanDatabaseSyncJob(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
detail, _ := json.Marshal(map[string]any{"tables": job.Tables, "warnings": job.Warnings, "errors": job.Errors})
|
||||
items = append(items, SystemLogItem{
|
||||
ID: job.ID,
|
||||
Category: "database_sync",
|
||||
Type: job.Direction,
|
||||
Target: directionLabel(job.Direction),
|
||||
Status: job.Status,
|
||||
Message: strings.Join(job.Output, "\n"),
|
||||
Detail: string(detail),
|
||||
CreatedAt: firstNonEmpty(job.FinishedAt, job.StartedAt),
|
||||
})
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) legacySyncLogs() ([]SystemLogItem, error) {
|
||||
rows, err := s.query(`SELECT id, status, summary, stats_json, started_at, finished_at FROM legacy_sync_jobs ORDER BY id DESC LIMIT 500`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items := []SystemLogItem{}
|
||||
for rows.Next() {
|
||||
var item LegacySyncJob
|
||||
if err := rows.Scan(&item.ID, &item.Status, &item.Summary, &item.StatsJSON, &item.StartedAt, &item.FinishedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, SystemLogItem{
|
||||
ID: item.ID,
|
||||
Category: "legacy_sync",
|
||||
Type: "legacy.sync",
|
||||
Target: "legacy",
|
||||
Status: item.Status,
|
||||
Message: item.Summary,
|
||||
Detail: item.StatsJSON,
|
||||
CreatedAt: firstNonEmpty(item.FinishedAt, item.StartedAt),
|
||||
})
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func matchesSystemLog(filters SystemLogFilters, item SystemLogItem, category string) bool {
|
||||
if value := strings.TrimSpace(filters.Category); value != "" && value != category && value != item.Category {
|
||||
return false
|
||||
}
|
||||
if value := strings.ToLower(strings.TrimSpace(filters.Query)); value != "" {
|
||||
haystack := strings.ToLower(strings.Join([]string{item.Category, item.Type, item.Target, item.Status, item.Message, item.Detail, item.CreatedAt}, " "))
|
||||
return strings.Contains(haystack, value)
|
||||
}
|
||||
return true
|
||||
}
|
||||
Reference in New Issue
Block a user