服务端媒体源导入/保存/客户端输出链路修复:支持 snake/camel、subcategories/sources,默认客户端可见,保存后发布兼容 media-types.json。
build-winui / winui (push) Has been cancelled

新增数据库同步 Job API、持久化状态、实时输出、最新任务恢复,以及系统日志聚合接口。
管理端优化:日志中心、运维实时状态框、同步输出自动滚动、仪表盘“输出”列、真实延迟空态、本地 favicon/avatar。
新增 server/unified-management/assets/favicon.ico 和 developer-avatar.png,并接好 /favicon.ico、/admin/favicon.ico、/setup/favicon.ico、/assets/*。
WinUI 随机放映室卡片优先显示子接口原始 Description。
Inno 安装器输出框改为选区末尾 + SendMessage 滚动到底部。
This commit is contained in:
QWQLwToo
2026-06-29 22:28:58 +08:00
parent f00124c1c0
commit 7745e7a2d4
36 changed files with 1482 additions and 153 deletions
@@ -33,10 +33,10 @@ func NormalizeBranding(current BrandingConfig, incoming BrandingConfig) Branding
next.FeedbackEmail = value
}
if next.SiteIconURL == "" {
next.SiteIconURL = "https://img.ymhut.cn/file/1782108850041_icon.webp"
next.SiteIconURL = "/assets/favicon.ico"
}
if next.DeveloperAvatarURL == "" {
next.DeveloperAvatarURL = "https://img.ymhut.cn/file/1782108780690_b_3db45f3787f19192c8de8e06bc0987ef.webp"
next.DeveloperAvatarURL = "/assets/developer-avatar.png"
}
if next.DeveloperName == "" {
next.DeveloperName = "YMhut"
@@ -142,7 +142,7 @@ func defaults(root string) *Config {
TimestampWindowSeconds: 600,
MaxRequestBytes: 12 * 1024 * 1024,
MaxPackageBytes: 10 * 1024 * 1024,
SourceCheckSeconds: 300,
SourceCheckSeconds: 60,
Database: DatabaseConfig{
Provider: "sqlite",
SQLitePath: filepath.Join(root, "storage", "unified.sqlite"),
@@ -150,7 +150,7 @@ func defaults(root string) *Config {
MySQLPort: 3306,
FailoverEnabled: true,
HotSyncEnabled: true,
HealthIntervalSec: 30,
HealthIntervalSec: 60,
MaxOpenConns: 10,
MaxIdleConns: 4,
ConnMaxLifetimeSeconds: 300,
@@ -163,8 +163,8 @@ func defaults(root string) *Config {
TimeoutSeconds: 20,
},
Branding: BrandingConfig{
SiteIconURL: "https://img.ymhut.cn/file/1782108850041_icon.webp",
DeveloperAvatarURL: "https://img.ymhut.cn/file/1782108780690_b_3db45f3787f19192c8de8e06bc0987ef.webp",
SiteIconURL: "/assets/favicon.ico",
DeveloperAvatarURL: "/assets/developer-avatar.png",
DeveloperName: "YMhut",
FeedbackEmail: "support@ymhut.cn",
},
@@ -393,7 +393,7 @@ func normalize(root string, cfg *Config) {
}
}
if cfg.Database.HealthIntervalSec <= 0 {
cfg.Database.HealthIntervalSec = 30
cfg.Database.HealthIntervalSec = 60
}
if cfg.Database.MaxOpenConns <= 0 {
cfg.Database.MaxOpenConns = 10
@@ -435,7 +435,7 @@ func normalize(root string, cfg *Config) {
cfg.UploadGuard.MaxReadableTextBytes = 256 * 1024
}
if cfg.SourceCheckSeconds <= 0 {
cfg.SourceCheckSeconds = 300
cfg.SourceCheckSeconds = 60
}
if cfg.Mail.Port <= 0 {
cfg.Mail.Port = 465
@@ -454,10 +454,10 @@ func normalize(root string, cfg *Config) {
cfg.Mail.TimeoutSeconds = 20
}
if cfg.Branding.SiteIconURL == "" {
cfg.Branding.SiteIconURL = "https://img.ymhut.cn/file/1782108850041_icon.webp"
cfg.Branding.SiteIconURL = "/assets/favicon.ico"
}
if cfg.Branding.DeveloperAvatarURL == "" {
cfg.Branding.DeveloperAvatarURL = "https://img.ymhut.cn/file/1782108780690_b_3db45f3787f19192c8de8e06bc0987ef.webp"
cfg.Branding.DeveloperAvatarURL = "/assets/developer-avatar.png"
}
if cfg.Branding.DeveloperName == "" {
cfg.Branding.DeveloperName = "YMhut"
@@ -21,31 +21,80 @@ func (s *Store) CopyRemoteToSQLite() (string, error) {
}
func (s *Store) ImportSQLiteToRemote() (SyncResult, error) {
if !s.trySyncLock() {
return SyncResult{Direction: "sqlite_to_remote", Status: "running", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"database sync is already running"}}, errors.New("database sync is already running")
}
defer s.syncMu.Unlock()
s.mu.RLock()
remote := s.remoteDB
remoteDialect := s.remoteDialect
local := s.localDB
localDialect := s.localDialect
s.mu.RUnlock()
if remote == nil {
result := SyncResult{Direction: "sqlite_to_remote", Status: "skipped", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"remote database is not configured"}}
s.setSyncStatus(result, nil)
return result, nil
}
result, err := copyAllTables(local, localDialect, remote, remoteDialect, "sqlite_to_remote")
s.setSyncStatus(result, err)
return result, err
return s.RunDatabaseSync("sqlite_to_remote")
}
func (s *Store) SyncNow() (SyncResult, error) {
return s.RunDatabaseSync("remote_to_sqlite")
}
func (s *Store) QueueDatabaseSync(direction string) (DatabaseSyncJob, error) {
normalized, err := normalizeSyncDirection(direction)
if err != nil {
return DatabaseSyncJob{}, err
}
if !s.trySyncLock() {
return SyncResult{Direction: "remote_to_sqlite", Status: "running", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"database sync is already running"}}, errors.New("database sync is already running")
job, insertErr := s.insertDatabaseSyncJob(normalized, "skipped", []string{"同步任务未启动:已有数据库同步正在执行。"}, []string{"database sync is already running"}, nil, nil, Now())
if insertErr != nil {
return DatabaseSyncJob{}, insertErr
}
return job, errors.New("database sync is already running")
}
job, err := s.insertDatabaseSyncJob(normalized, "running", []string{"同步任务已创建,正在准备数据库连接。"}, nil, nil, map[string]int{}, "")
if err != nil {
s.syncMu.Unlock()
return DatabaseSyncJob{}, err
}
s.setCurrentSyncJob(&job)
go func(jobID int64, jobDirection string) {
defer s.syncMu.Unlock()
_, _ = s.runDatabaseSyncLocked(jobID, jobDirection)
}(job.ID, normalized)
return job, nil
}
func (s *Store) RunDatabaseSync(direction string) (SyncResult, error) {
normalized, err := normalizeSyncDirection(direction)
if err != nil {
return SyncResult{}, err
}
if !s.trySyncLock() {
result := SyncResult{
Direction: normalized,
Status: "skipped",
Skipped: true,
StartedAt: Now(),
FinishedAt: Now(),
Tables: map[string]int{},
Warnings: []string{"database sync is already running"},
Output: []string{"同步任务未启动:已有数据库同步正在执行。"},
}
_, _ = s.insertDatabaseSyncJob(normalized, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
s.setSyncStatus(result, errors.New("database sync is already running"))
return result, errors.New("database sync is already running")
}
defer s.syncMu.Unlock()
job, err := s.insertDatabaseSyncJob(normalized, "running", []string{"同步任务已创建,正在准备数据库连接。"}, nil, nil, map[string]int{}, "")
if err != nil {
return SyncResult{}, err
}
s.setCurrentSyncJob(&job)
return s.runDatabaseSyncLocked(job.ID, normalized)
}
func (s *Store) runDatabaseSyncLocked(jobID int64, direction string) (SyncResult, error) {
started := Now()
if job, err := s.GetDatabaseSyncJob(jobID); err == nil && job.StartedAt != "" {
started = job.StartedAt
}
result := SyncResult{
Direction: direction,
Status: "completed",
StartedAt: started,
Tables: map[string]int{},
Output: []string{"同步开始:" + directionLabel(direction)},
}
_ = s.updateDatabaseSyncJob(jobID, "running", result.Output, result.Warnings, result.Errors, result.Tables, "")
s.mu.RLock()
remote := s.remoteDB
remoteDialect := s.remoteDialect
@@ -53,18 +102,43 @@ func (s *Store) SyncNow() (SyncResult, error) {
localDialect := s.localDialect
s.mu.RUnlock()
if remote == nil {
result := SyncResult{Direction: "remote_to_sqlite", Status: "skipped", Skipped: true, Tables: map[string]int{}, FinishedAt: Now(), Warnings: []string{"remote database is not configured"}}
result.Status = "skipped"
result.Skipped = true
result.Warnings = append(result.Warnings, "remote database is not configured")
result.Output = append(result.Output, "远程 MySQL 未配置或不可用,未执行数据复制。")
result.FinishedAt = Now()
_ = s.updateDatabaseSyncJob(jobID, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
s.setSyncStatus(result, nil)
return result, nil
}
result, err := copyAllTables(remote, remoteDialect, local, localDialect, "remote_to_sqlite")
s.setSyncStatus(result, err)
return result, err
src, srcDialect, dst, dstDialect := local, localDialect, remote, remoteDialect
if direction == "remote_to_sqlite" {
src, srcDialect, dst, dstDialect = remote, remoteDialect, local, localDialect
}
err := copyAllTablesWithProgress(src, srcDialect, dst, dstDialect, direction, func(table string, count int) {
result.Tables[table] = count
result.Output = append(result.Output, fmt.Sprintf("%s%d 条", table, count))
_ = s.updateDatabaseSyncJob(jobID, "running", result.Output, result.Warnings, result.Errors, result.Tables, "")
})
result.FinishedAt = Now()
if err != nil {
result.Status = "failed"
result.Errors = append(result.Errors, err.Error())
result.Output = append(result.Output, "同步失败:"+err.Error())
_ = s.updateDatabaseSyncJob(jobID, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
s.setSyncStatus(result, err)
return result, err
}
result.Output = append(result.Output, "同步完成:"+directionLabel(direction))
_ = s.updateDatabaseSyncJob(jobID, result.Status, result.Output, result.Warnings, result.Errors, result.Tables, result.FinishedAt)
s.setSyncStatus(result, nil)
return result, nil
}
func (s *Store) setSyncStatus(result SyncResult, err error) {
s.mu.Lock()
defer s.mu.Unlock()
s.status.CurrentSyncJob = nil
if err != nil {
s.status.LastSyncAt = result.FinishedAt
s.status.LastSyncError = err.Error()
@@ -78,6 +152,33 @@ func (s *Store) trySyncLock() bool {
return s.syncMu.TryLock()
}
func (s *Store) setCurrentSyncJob(job *DatabaseSyncJob) {
s.mu.Lock()
defer s.mu.Unlock()
s.status.CurrentSyncJob = job
}
func normalizeSyncDirection(value string) (string, error) {
switch strings.ToLower(strings.TrimSpace(value)) {
case "sqlite_to_remote", "import", "sqlite_to_mysql":
return "sqlite_to_remote", nil
case "remote_to_sqlite", "sync", "mysql_to_sqlite":
return "remote_to_sqlite", nil
default:
return "", errors.New("unsupported database sync direction")
}
}
func directionLabel(direction string) string {
if direction == "sqlite_to_remote" {
return "SQLite -> MySQL"
}
if direction == "remote_to_sqlite" {
return "MySQL -> SQLite"
}
return direction
}
type tableSpec struct {
Name string
Columns []string
@@ -100,6 +201,7 @@ var syncTables = []tableSpec{
{"source_endpoints", []string{"id", "category_id", "category_name", "source_id", "name", "description", "method", "api_url", "url_template", "thumbnail_url", "proxy_mode", "timeout_ms", "retry_count", "cache_seconds", "check_interval_sec", "enabled", "client_visible", "supported_formats", "last_status", "last_latency_ms", "last_checked_at", "last_error", "consecutive_failure", "created_at", "updated_at"}, []string{"id"}},
{"endpoint_health_checks", []string{"id", "source_db_id", "status", "latency_ms", "error", "checked_at"}, []string{"id"}},
{"endpoint_call_logs", []string{"id", "source_id", "status", "latency_ms", "error", "client", "created_at"}, []string{"id"}},
{"database_sync_jobs", []string{"id", "direction", "status", "message", "tables_json", "started_at", "finished_at"}, []string{"id"}},
{"system_settings", []string{"key", "value", "updated_at"}, []string{"key"}},
{"audit_logs", []string{"id", "actor", "type", "target", "message", "ip", "user_agent", "created_at"}, []string{"id"}},
{"legacy_json_revisions", []string{"id", "name", "raw", "note", "created_by", "created_at"}, []string{"id"}},
@@ -109,19 +211,31 @@ var syncTables = []tableSpec{
func copyAllTables(src *sql.DB, srcDialect dialect, dst *sql.DB, dstDialect dialect, direction string) (SyncResult, error) {
result := SyncResult{Direction: direction, Status: "completed", Tables: map[string]int{}, FinishedAt: Now()}
for _, table := range syncTables {
count, err := copyTable(src, srcDialect, dst, dstDialect, table)
if err != nil {
result.Status = "failed"
result.FinishedAt = Now()
return result, err
}
result.Tables[table.Name] = count
err := copyAllTablesWithProgress(src, srcDialect, dst, dstDialect, direction, func(table string, count int) {
result.Tables[table] = count
})
if err != nil {
result.Status = "failed"
result.FinishedAt = Now()
return result, err
}
result.FinishedAt = Now()
return result, nil
}
func copyAllTablesWithProgress(src *sql.DB, srcDialect dialect, dst *sql.DB, dstDialect dialect, direction string, progress func(table string, count int)) error {
for _, table := range syncTables {
count, err := copyTable(src, srcDialect, dst, dstDialect, table)
if err != nil {
return err
}
if progress != nil {
progress(table.Name, count)
}
}
return nil
}
func copyTable(src *sql.DB, srcDialect dialect, dst *sql.DB, dstDialect dialect, spec tableSpec) (int, error) {
rows, err := src.Query(srcDialect.rebind("SELECT " + srcDialect.columnList(spec.Columns) + " FROM " + spec.Name))
if err != nil {
@@ -0,0 +1,141 @@
package db
import (
"database/sql"
"encoding/json"
"errors"
"strings"
)
type syncJobMessage struct {
Output []string `json:"output"`
Warnings []string `json:"warnings"`
Errors []string `json:"errors"`
}
type syncJobScanner interface {
Scan(dest ...any) error
}
func (s *Store) insertDatabaseSyncJob(direction, status string, output, warnings, jobErrors []string, tables map[string]int, finishedAt string) (DatabaseSyncJob, error) {
startedAt := Now()
message, tablesJSON := encodeDatabaseSyncJob(output, warnings, jobErrors, tables)
id, err := s.insertID(`INSERT INTO database_sync_jobs (direction, status, message, tables_json, started_at, finished_at) VALUES (?, ?, ?, ?, ?, ?)`,
direction, status, message, tablesJSON, startedAt, finishedAt)
if err != nil {
return DatabaseSyncJob{}, err
}
return s.GetDatabaseSyncJob(id)
}
func (s *Store) updateDatabaseSyncJob(id int64, status string, output, warnings, jobErrors []string, tables map[string]int, finishedAt string) error {
message, tablesJSON := encodeDatabaseSyncJob(output, warnings, jobErrors, tables)
if _, err := s.exec(`UPDATE database_sync_jobs SET status = ?, message = ?, tables_json = ?, finished_at = ? WHERE id = ?`,
status, message, tablesJSON, finishedAt, id); err != nil {
return err
}
if job, err := s.GetDatabaseSyncJob(id); err == nil {
s.mu.Lock()
if s.status.CurrentSyncJob != nil && s.status.CurrentSyncJob.ID == id {
s.status.CurrentSyncJob = &job
}
s.mu.Unlock()
}
return nil
}
func (s *Store) GetDatabaseSyncJob(id int64) (DatabaseSyncJob, error) {
job, err := scanDatabaseSyncJob(s.queryRow(`SELECT id, direction, status, message, tables_json, started_at, finished_at FROM database_sync_jobs WHERE id = ?`, id))
if errors.Is(err, sql.ErrNoRows) {
return DatabaseSyncJob{}, errors.New("database sync job not found")
}
return job, err
}
func (s *Store) LatestDatabaseSyncJob() (DatabaseSyncJob, error) {
job, err := scanDatabaseSyncJob(s.queryRow(`SELECT id, direction, status, message, tables_json, started_at, finished_at FROM database_sync_jobs ORDER BY id DESC LIMIT 1`))
if errors.Is(err, sql.ErrNoRows) {
return DatabaseSyncJob{}, errors.New("database sync job not found")
}
return job, err
}
func (s *Store) restoreDatabaseSyncStatus() {
job, err := s.LatestDatabaseSyncJob()
if err != nil {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if job.Status == "running" {
s.status.CurrentSyncJob = &job
} else {
s.status.CurrentSyncJob = nil
}
if job.FinishedAt != "" {
s.status.LastSyncAt = job.FinishedAt
} else {
s.status.LastSyncAt = job.StartedAt
}
if job.Status == "failed" {
s.status.LastSyncError = strings.Join(job.Errors, "; ")
} else {
s.status.LastSyncError = ""
}
}
func scanDatabaseSyncJob(scanner syncJobScanner) (DatabaseSyncJob, error) {
var job DatabaseSyncJob
var message, tablesJSON string
if err := scanner.Scan(&job.ID, &job.Direction, &job.Status, &message, &tablesJSON, &job.StartedAt, &job.FinishedAt); err != nil {
return DatabaseSyncJob{}, err
}
decodeDatabaseSyncJob(message, tablesJSON, &job)
return job, nil
}
func encodeDatabaseSyncJob(output, warnings, jobErrors []string, tables map[string]int) (string, string) {
if output == nil {
output = []string{}
}
if warnings == nil {
warnings = []string{}
}
if jobErrors == nil {
jobErrors = []string{}
}
if tables == nil {
tables = map[string]int{}
}
message, _ := json.Marshal(syncJobMessage{Output: output, Warnings: warnings, Errors: jobErrors})
tableData, _ := json.Marshal(tables)
return string(message), string(tableData)
}
func decodeDatabaseSyncJob(message, tablesJSON string, job *DatabaseSyncJob) {
job.Output = []string{}
job.Warnings = []string{}
job.Errors = []string{}
job.Tables = map[string]int{}
var payload syncJobMessage
if strings.TrimSpace(message) != "" && json.Unmarshal([]byte(message), &payload) == nil {
job.Output = payload.Output
job.Warnings = payload.Warnings
job.Errors = payload.Errors
} else if strings.TrimSpace(message) != "" {
job.Output = []string{message}
}
_ = json.Unmarshal([]byte(tablesJSON), &job.Tables)
if job.Output == nil {
job.Output = []string{}
}
if job.Warnings == nil {
job.Warnings = []string{}
}
if job.Errors == nil {
job.Errors = []string{}
}
if job.Tables == nil {
job.Tables = map[string]int{}
}
}
+56 -11
View File
@@ -20,17 +20,20 @@ type adminRow struct {
}
type DatabaseStatus struct {
ActiveProvider string `json:"activeProvider"`
ConfigProvider string `json:"configProvider"`
SchemaVersion string `json:"schemaVersion"`
SQLiteReady bool `json:"sqliteReady"`
RemoteReady bool `json:"remoteReady"`
FailoverActive bool `json:"failoverActive"`
LastError string `json:"lastError"`
LastFailoverAt string `json:"lastFailoverAt"`
LastRecoveredAt string `json:"lastRecoveredAt"`
LastSyncAt string `json:"lastSyncAt"`
LastSyncError string `json:"lastSyncError"`
ActiveProvider string `json:"activeProvider"`
ConfigProvider string `json:"configProvider"`
SchemaVersion string `json:"schemaVersion"`
SQLiteReady bool `json:"sqliteReady"`
RemoteReady bool `json:"remoteReady"`
FailoverActive bool `json:"failoverActive"`
LastError string `json:"lastError"`
LastFailoverAt string `json:"lastFailoverAt"`
LastRecoveredAt string `json:"lastRecoveredAt"`
LastSyncAt string `json:"lastSyncAt"`
LastSyncError string `json:"lastSyncError"`
LastHealthCheckedAt string `json:"lastHealthCheckedAt"`
LastHealthStatus string `json:"lastHealthStatus"`
CurrentSyncJob *DatabaseSyncJob `json:"currentSyncJob,omitempty"`
}
type SyncResult struct {
@@ -38,10 +41,25 @@ type SyncResult struct {
Status string `json:"status"`
Skipped bool `json:"skipped"`
Warnings []string `json:"warnings,omitempty"`
Errors []string `json:"errors,omitempty"`
Output []string `json:"output,omitempty"`
StartedAt string `json:"startedAt,omitempty"`
Tables map[string]int `json:"tables"`
FinishedAt string `json:"finishedAt"`
}
type DatabaseSyncJob struct {
ID int64 `json:"id"`
Direction string `json:"direction"`
Status string `json:"status"`
Output []string `json:"output"`
Warnings []string `json:"warnings"`
Errors []string `json:"errors"`
Tables map[string]int `json:"tables"`
StartedAt string `json:"startedAt"`
FinishedAt string `json:"finishedAt"`
}
type AdminUser struct {
ID int64 `json:"id"`
Username string `json:"username"`
@@ -246,6 +264,8 @@ type Source struct {
ConsecutiveFailure int `json:"consecutiveFailure"`
CreatedAt string `json:"createdAt"`
UpdatedAt string `json:"updatedAt"`
EnabledSet bool `json:"-"`
ClientVisibleSet bool `json:"-"`
}
type SourceCheck struct {
@@ -301,3 +321,28 @@ type LegacyJsonRevision struct {
CreatedBy string `json:"createdBy"`
CreatedAt string `json:"createdAt"`
}
type SystemLogItem struct {
ID int64 `json:"id"`
Category string `json:"category"`
Type string `json:"type"`
Target string `json:"target"`
Status string `json:"status"`
Message string `json:"message"`
Detail string `json:"detail"`
CreatedAt string `json:"createdAt"`
}
type SystemLogPage struct {
Items []SystemLogItem `json:"items"`
Total int `json:"total"`
Page int `json:"page"`
PerPage int `json:"perPage"`
}
type SystemLogFilters struct {
Page int
PerPage int
Category string
Query string
}
@@ -10,6 +10,8 @@ func (s *Store) UpsertSource(item Source) (Source, error) {
if item.SourceID == "" {
item.SourceID = item.CategoryID + "-" + item.Name
}
existing, existingErr := s.GetSourceBySourceID(item.SourceID)
hasExisting := existingErr == nil
if item.Method == "" {
item.Method = "GET"
}
@@ -33,10 +35,30 @@ func (s *Store) UpsertSource(item Source) (Source, error) {
item.CheckIntervalSec = item.CacheSeconds
}
if item.SupportedFormats == "" {
item.SupportedFormats = "[]"
if hasExisting && existing.SupportedFormats != "" {
item.SupportedFormats = existing.SupportedFormats
} else {
item.SupportedFormats = "[]"
}
}
if item.LastStatus == "" {
item.LastStatus = "unknown"
if hasExisting && existing.LastStatus != "" {
item.LastStatus = existing.LastStatus
} else {
item.LastStatus = "unknown"
}
}
if item.LastLatencyMS == 0 && hasExisting {
item.LastLatencyMS = existing.LastLatencyMS
}
if item.LastCheckedAt == "" && hasExisting {
item.LastCheckedAt = existing.LastCheckedAt
}
if item.LastError == "" && hasExisting {
item.LastError = existing.LastError
}
if item.ConsecutiveFailure == 0 && hasExisting {
item.ConsecutiveFailure = existing.ConsecutiveFailure
}
if item.CategoryID == "" {
item.CategoryID = "custom"
@@ -44,6 +66,21 @@ func (s *Store) UpsertSource(item Source) (Source, error) {
if item.CategoryName == "" {
item.CategoryName = item.CategoryID
}
if !item.EnabledSet {
item.Enabled = true
if hasExisting {
item.Enabled = existing.Enabled
}
}
if !item.ClientVisibleSet {
item.ClientVisible = true
if hasExisting {
item.ClientVisible = existing.ClientVisible
}
}
if item.CreatedAt == "" && hasExisting {
item.CreatedAt = existing.CreatedAt
}
_, _ = s.exec(`INSERT INTO source_categories (category_id, name, enabled, ui_config, created_at, updated_at)
VALUES (?, ?, 1, '{}', ?, ?)
ON CONFLICT (category_id) DO UPDATE SET name = excluded.name, updated_at = excluded.updated_at`,
@@ -97,6 +134,12 @@ func (s *Store) CountSources() (int, error) {
return count, err
}
func (s *Store) CountClientVisibleSources() (int, error) {
var count int
err := s.queryRow(`SELECT COUNT(*) FROM source_endpoints WHERE enabled = 1 AND client_visible = 1`).Scan(&count)
return count, err
}
func (s *Store) DeleteSource(sourceID string) error {
_, err := s.exec(`DELETE FROM source_endpoints WHERE source_id = ?`, sourceID)
return err
+21 -1
View File
@@ -69,6 +69,7 @@ func Open(cfg *config.Config) (*Store, error) {
SchemaVersion: CurrentSchemaVersion,
SQLiteReady: true,
LastRecoveredAt: Now(),
LastHealthStatus: "not_configured",
},
}
if err := store.migrate(local, localDialect); err != nil {
@@ -86,6 +87,7 @@ func Open(cfg *config.Config) (*Store, error) {
store.markFailover(err)
}
}
store.restoreDatabaseSyncStatus()
go store.maintain()
return store, nil
}
@@ -171,14 +173,17 @@ func (s *Store) ReconfigureDatabase(cfg *config.Config) error {
s.status.RemoteReady = remote != nil
s.status.LastError = ""
s.status.FailoverActive = false
s.status.LastHealthCheckedAt = Now()
if remote != nil {
s.db = remote
s.dialect = remoteDialect
s.status.ActiveProvider = "mysql"
s.status.LastHealthStatus = "ok"
} else {
s.db = local
s.dialect = localDialect
s.status.ActiveProvider = "sqlite"
s.status.LastHealthStatus = "not_configured"
}
s.status.LastRecoveredAt = Now()
s.mu.Unlock()
@@ -188,6 +193,7 @@ func (s *Store) ReconfigureDatabase(cfg *config.Config) error {
if oldLocal != nil && oldLocal != local {
_ = oldLocal.Close()
}
s.restoreDatabaseSyncStatus()
return nil
}
@@ -237,7 +243,11 @@ func (s *Store) insertID(query string, args ...any) (int64, error) {
}
func (s *Store) maintain() {
ticker := time.NewTicker(time.Duration(s.cfg.Database.HealthIntervalSec) * time.Second)
interval := s.cfg.Database.HealthIntervalSec
if interval <= 0 {
interval = 60
}
ticker := time.NewTicker(time.Duration(interval) * time.Second)
defer ticker.Stop()
for {
select {
@@ -276,12 +286,18 @@ func (s *Store) openRemote() error {
s.status.FailoverActive = false
s.status.LastError = ""
s.status.LastRecoveredAt = Now()
s.status.LastHealthCheckedAt = Now()
s.status.LastHealthStatus = "ok"
s.mu.Unlock()
return nil
}
func (s *Store) checkRemote() {
if !strings.EqualFold(s.cfg.Database.Provider, "mysql") {
s.mu.Lock()
s.status.LastHealthCheckedAt = Now()
s.status.LastHealthStatus = "not_configured"
s.mu.Unlock()
return
}
s.mu.RLock()
@@ -311,6 +327,8 @@ func (s *Store) checkRemote() {
s.status.FailoverActive = false
s.status.LastError = ""
s.status.LastRecoveredAt = Now()
s.status.LastHealthCheckedAt = Now()
s.status.LastHealthStatus = "ok"
s.mu.Unlock()
}
@@ -329,4 +347,6 @@ func (s *Store) markFailover(err error) {
s.status.FailoverActive = !strings.EqualFold(s.cfg.Database.Provider, "sqlite")
s.status.LastError = err.Error()
s.status.LastFailoverAt = Now()
s.status.LastHealthCheckedAt = Now()
s.status.LastHealthStatus = "error"
}
@@ -391,3 +391,47 @@ func TestDashboardOverviewKeepsChecksForDeletedSources(t *testing.T) {
t.Fatalf("deleted source check should have fallback sourceId/name: %#v", checks[0])
}
}
func TestDatabaseSyncJobPersistsLatestStatusAndOutput(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "unified.sqlite")
store, err := Open(&config.Config{
StorageDir: root,
Database: config.DatabaseConfig{
Provider: "sqlite",
SQLitePath: path,
FailoverEnabled: true,
HealthIntervalSec: 3600,
MaxOpenConns: 1,
MaxIdleConns: 1,
ConnMaxLifetimeSeconds: 60,
},
})
if err != nil {
t.Fatal(err)
}
defer store.Close()
result, err := store.ImportSQLiteToRemote()
if err != nil {
t.Fatal(err)
}
if !result.Skipped || result.Status != "skipped" {
t.Fatalf("expected missing remote sync to be skipped, got %#v", result)
}
latest, err := store.LatestDatabaseSyncJob()
if err != nil {
t.Fatal(err)
}
if latest.Direction != "sqlite_to_remote" || latest.Status != "skipped" {
t.Fatalf("unexpected latest sync job: %#v", latest)
}
if len(latest.Output) == 0 || latest.Tables == nil {
t.Fatalf("latest job should include output and tables: %#v", latest)
}
store.restoreDatabaseSyncStatus()
status := store.Status()
if status.LastSyncAt == "" || status.LastSyncError != "" {
t.Fatalf("sync status was not restored from latest job: %#v", status)
}
}
@@ -0,0 +1,227 @@
package db
import (
"encoding/json"
"fmt"
"sort"
"strings"
)
func (s *Store) ListSystemLogsPage(filters SystemLogFilters) (SystemLogPage, error) {
page := filters.Page
if page <= 0 {
page = 1
}
perPage := filters.PerPage
if perPage <= 0 {
perPage = 35
}
if perPage > 100 {
perPage = 100
}
items, err := s.collectSystemLogs(filters)
if err != nil {
return SystemLogPage{}, err
}
sort.SliceStable(items, func(i, j int) bool {
if items[i].CreatedAt == items[j].CreatedAt {
return items[i].ID > items[j].ID
}
return items[i].CreatedAt > items[j].CreatedAt
})
total := len(items)
start := (page - 1) * perPage
if start > total {
start = total
}
end := start + perPage
if end > total {
end = total
}
return SystemLogPage{Items: items[start:end], Total: total, Page: page, PerPage: perPage}, nil
}
func (s *Store) collectSystemLogs(filters SystemLogFilters) ([]SystemLogItem, error) {
items := []SystemLogItem{}
appendItems := func(category string, next []SystemLogItem, err error) error {
if err != nil {
return err
}
for _, item := range next {
if matchesSystemLog(filters, item, category) {
items = append(items, item)
}
}
return nil
}
logs, err := s.operationLogs()
if err := appendItems("operation", logs, err); err != nil {
return nil, err
}
logs, err = s.healthLogs()
if err := appendItems("health", logs, err); err != nil {
return nil, err
}
logs, err = s.clientCallLogs()
if err := appendItems("client", logs, err); err != nil {
return nil, err
}
logs, err = s.databaseSyncLogs()
if err := appendItems("database_sync", logs, err); err != nil {
return nil, err
}
logs, err = s.legacySyncLogs()
if err := appendItems("legacy_sync", logs, err); err != nil {
return nil, err
}
return items, nil
}
func (s *Store) operationLogs() ([]SystemLogItem, error) {
rows, err := s.query(`SELECT id, actor, type, target, message, ip, user_agent, created_at FROM audit_logs ORDER BY id DESC LIMIT 500`)
if err != nil {
return nil, err
}
defer rows.Close()
items := []SystemLogItem{}
for rows.Next() {
var item AuditLog
if err := rows.Scan(&item.ID, &item.Actor, &item.Type, &item.Target, &item.Message, &item.IP, &item.UserAgent, &item.CreatedAt); err != nil {
return nil, err
}
items = append(items, SystemLogItem{
ID: item.ID,
Category: "operation",
Type: item.Type,
Target: item.Target,
Status: firstNonEmpty(item.Actor, "system"),
Message: item.Message,
Detail: strings.TrimSpace(item.IP + " " + item.UserAgent),
CreatedAt: item.CreatedAt,
})
}
return items, rows.Err()
}
func (s *Store) healthLogs() ([]SystemLogItem, error) {
rows, err := s.query(`SELECT h.id, h.source_db_id, COALESCE(e.source_id, ''), COALESCE(e.name, ''), h.status, h.latency_ms, h.error, h.checked_at
FROM endpoint_health_checks h LEFT JOIN source_endpoints e ON e.id = h.source_db_id
ORDER BY h.id DESC LIMIT 500`)
if err != nil {
return nil, err
}
defer rows.Close()
items := []SystemLogItem{}
for rows.Next() {
var id, sourceDBID int64
var sourceID, name, status, message, checkedAt string
var latency int
if err := rows.Scan(&id, &sourceDBID, &sourceID, &name, &status, &latency, &message, &checkedAt); err != nil {
return nil, err
}
target := firstNonEmpty(sourceID, fmt.Sprintf("source#%d", sourceDBID))
items = append(items, SystemLogItem{
ID: id,
Category: "health",
Type: "endpoint.health",
Target: target,
Status: status,
Message: firstNonEmpty(name, target),
Detail: fmt.Sprintf("%dms %s", latency, message),
CreatedAt: checkedAt,
})
}
return items, rows.Err()
}
func (s *Store) clientCallLogs() ([]SystemLogItem, error) {
rows, err := s.query(`SELECT id, source_id, status, latency_ms, error, client, created_at FROM endpoint_call_logs ORDER BY id DESC LIMIT 500`)
if err != nil {
return nil, err
}
defer rows.Close()
items := []SystemLogItem{}
for rows.Next() {
var id int64
var sourceID, status, message, client, createdAt string
var latency int
if err := rows.Scan(&id, &sourceID, &status, &latency, &message, &client, &createdAt); err != nil {
return nil, err
}
items = append(items, SystemLogItem{
ID: id,
Category: "client",
Type: "endpoint.call",
Target: sourceID,
Status: status,
Message: message,
Detail: fmt.Sprintf("%dms %s", latency, client),
CreatedAt: createdAt,
})
}
return items, rows.Err()
}
func (s *Store) databaseSyncLogs() ([]SystemLogItem, error) {
rows, err := s.query(`SELECT id, direction, status, message, tables_json, started_at, finished_at FROM database_sync_jobs ORDER BY id DESC LIMIT 500`)
if err != nil {
return nil, err
}
defer rows.Close()
items := []SystemLogItem{}
for rows.Next() {
job, err := scanDatabaseSyncJob(rows)
if err != nil {
return nil, err
}
detail, _ := json.Marshal(map[string]any{"tables": job.Tables, "warnings": job.Warnings, "errors": job.Errors})
items = append(items, SystemLogItem{
ID: job.ID,
Category: "database_sync",
Type: job.Direction,
Target: directionLabel(job.Direction),
Status: job.Status,
Message: strings.Join(job.Output, "\n"),
Detail: string(detail),
CreatedAt: firstNonEmpty(job.FinishedAt, job.StartedAt),
})
}
return items, rows.Err()
}
func (s *Store) legacySyncLogs() ([]SystemLogItem, error) {
rows, err := s.query(`SELECT id, status, summary, stats_json, started_at, finished_at FROM legacy_sync_jobs ORDER BY id DESC LIMIT 500`)
if err != nil {
return nil, err
}
defer rows.Close()
items := []SystemLogItem{}
for rows.Next() {
var item LegacySyncJob
if err := rows.Scan(&item.ID, &item.Status, &item.Summary, &item.StatsJSON, &item.StartedAt, &item.FinishedAt); err != nil {
return nil, err
}
items = append(items, SystemLogItem{
ID: item.ID,
Category: "legacy_sync",
Type: "legacy.sync",
Target: "legacy",
Status: item.Status,
Message: item.Summary,
Detail: item.StatsJSON,
CreatedAt: firstNonEmpty(item.FinishedAt, item.StartedAt),
})
}
return items, rows.Err()
}
func matchesSystemLog(filters SystemLogFilters, item SystemLogItem, category string) bool {
if value := strings.TrimSpace(filters.Category); value != "" && value != category && value != item.Category {
return false
}
if value := strings.ToLower(strings.TrimSpace(filters.Query)); value != "" {
haystack := strings.ToLower(strings.Join([]string{item.Category, item.Type, item.Target, item.Status, item.Message, item.Detail, item.CreatedAt}, " "))
return strings.Contains(haystack, value)
}
return true
}
@@ -61,7 +61,8 @@ type mediaCandidate struct {
}
type legacyMedia struct {
Categories []legacyCategory `json:"categories"`
Categories []legacyCategory `json:"categories"`
Sources []legacySubcategory `json:"sources"`
}
const maxSourceProbeBytes int64 = 2 * 1024 * 1024
@@ -70,20 +71,45 @@ var absoluteURLPattern = regexp.MustCompile(`https?://[^\s"'<>\\]+`)
type legacyCategory struct {
ID string `json:"id"`
CategoryID string `json:"categoryId"`
CategoryIDAlt string `json:"category_id"`
Name string `json:"name"`
Enabled *bool `json:"enabled"`
Subcategories []legacySubcategory `json:"subcategories"`
Sources []legacySubcategory `json:"sources"`
Endpoints []legacySubcategory `json:"endpoints"`
}
type legacySubcategory struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
APIURL string `json:"api_url"`
ThumbnailURL string `json:"thumbnail_url"`
RefreshInterval int `json:"refresh_interval"`
SupportedFormats []string `json:"supported_formats"`
Downloadable bool `json:"downloadable"`
ID string `json:"id"`
SourceID string `json:"sourceId"`
SourceIDAlt string `json:"source_id"`
Name string `json:"name"`
Description string `json:"description"`
Method string `json:"method"`
APIURL string `json:"api_url"`
APIURLAlt string `json:"apiUrl"`
URL string `json:"url"`
URLTemplate string `json:"urlTemplate"`
URLTemplateAlt string `json:"url_template"`
ThumbnailURL string `json:"thumbnail_url"`
ThumbnailURLAlt string `json:"thumbnailUrl"`
ProxyMode string `json:"proxy_mode"`
ProxyModeAlt string `json:"proxyMode"`
RefreshInterval int `json:"refresh_interval"`
RefreshIntervalAlt int `json:"refreshInterval"`
CacheSeconds int `json:"cacheSeconds"`
CheckIntervalSec int `json:"checkIntervalSec"`
TimeoutMS int `json:"timeoutMs"`
RetryCount int `json:"retryCount"`
SupportedFormats []string `json:"supported_formats"`
SupportedFormatsAlt []string `json:"supportedFormats"`
MediaType string `json:"mediaType"`
MediaTypeAlt string `json:"media_type"`
Enabled *bool `json:"enabled"`
ClientVisible *bool `json:"clientVisible"`
ClientVisibleAlt *bool `json:"client_visible"`
Downloadable bool `json:"downloadable"`
}
func NewService(cfg *config.Config, store *db.Store) *Service {
@@ -126,7 +152,11 @@ func (s *Service) ImportLegacyMediaTypesIfEmpty(ctx context.Context) error {
if err != nil {
return err
}
if count > 0 {
visible, err := s.store.CountClientVisibleSources()
if err != nil {
return err
}
if count > 0 && visible > 0 {
return nil
}
return s.ImportLegacyMediaTypes(ctx)
@@ -141,28 +171,54 @@ func (s *Service) ImportLegacyMediaTypes(ctx context.Context) error {
if err := json.Unmarshal(data, &legacy); err != nil {
return err
}
if len(legacy.Sources) > 0 && len(legacy.Categories) == 0 {
legacy.Categories = []legacyCategory{{ID: "media", Name: "Media", Sources: legacy.Sources}}
}
for _, category := range legacy.Categories {
for _, sub := range category.Subcategories {
if strings.TrimSpace(sub.APIURL) == "" {
categoryID := firstNonEmpty(category.ID, category.CategoryID, category.CategoryIDAlt, "media")
categoryName := defaultString(category.Name, categoryID)
for _, sub := range category.entries() {
apiURL := firstNonEmpty(sub.APIURL, sub.APIURLAlt, sub.URL, sub.URLTemplate, sub.URLTemplateAlt)
if strings.TrimSpace(apiURL) == "" {
continue
}
formats, _ := json.Marshal(sub.SupportedFormats)
formatsList := firstNonEmptySlice(sub.SupportedFormats, sub.SupportedFormatsAlt)
if len(formatsList) == 0 && firstNonEmpty(sub.MediaType, sub.MediaTypeAlt) != "" {
formatsList = []string{firstNonEmpty(sub.MediaType, sub.MediaTypeAlt)}
}
formats, _ := json.Marshal(formatsList)
enabled := legacyEnabled(category.Enabled)
if sub.Enabled != nil {
enabled = *sub.Enabled
}
visible := true
if sub.ClientVisible != nil {
visible = *sub.ClientVisible
} else if sub.ClientVisibleAlt != nil {
visible = *sub.ClientVisibleAlt
}
interval := firstPositive(sub.CheckIntervalSec, sub.RefreshInterval, sub.RefreshIntervalAlt, sub.CacheSeconds, 60)
cacheSeconds := firstPositive(sub.CacheSeconds, interval, 60)
_, err := s.store.UpsertSource(db.Source{
CategoryID: defaultString(category.ID, "media"),
CategoryName: defaultString(category.Name, category.ID),
SourceID: defaultString(sub.ID, category.ID+"-"+sub.Name),
Name: defaultString(sub.Name, sub.ID),
CategoryID: categoryID,
CategoryName: categoryName,
SourceID: defaultString(firstNonEmpty(sub.ID, sub.SourceID, sub.SourceIDAlt), categoryID+"-"+sub.Name),
Name: defaultString(sub.Name, firstNonEmpty(sub.ID, sub.SourceID, sub.SourceIDAlt)),
Description: sub.Description,
Method: "GET",
APIURL: sub.APIURL,
ThumbnailURL: sub.ThumbnailURL,
ProxyMode: "client_direct",
TimeoutMS: 8000,
Method: firstNonEmpty(sub.Method, "GET"),
APIURL: apiURL,
URLTemplate: firstNonEmpty(sub.URLTemplate, sub.URLTemplateAlt, apiURL),
ThumbnailURL: firstNonEmpty(sub.ThumbnailURL, sub.ThumbnailURLAlt),
ProxyMode: firstNonEmpty(sub.ProxyMode, sub.ProxyModeAlt, "client_direct"),
TimeoutMS: firstPositive(sub.TimeoutMS, 8000),
RetryCount: 1,
CheckIntervalSec: maxInt(sub.RefreshInterval, 300),
Enabled: legacyEnabled(category.Enabled),
ClientVisible: true,
CacheSeconds: cacheSeconds,
CheckIntervalSec: interval,
Enabled: enabled,
ClientVisible: visible,
SupportedFormats: string(formats),
EnabledSet: true,
ClientVisibleSet: true,
})
if err != nil {
return err
@@ -229,23 +285,38 @@ func (s *Service) Catalog(includeHidden bool) (map[string]any, error) {
_ = json.Unmarshal([]byte(item.SupportedFormats), &formats)
sub := map[string]any{
"id": item.SourceID,
"sourceId": item.SourceID,
"source_id": item.SourceID,
"name": item.Name,
"description": item.Description,
"api_url": item.APIURL,
"apiUrl": item.APIURL,
"urlTemplate": firstNonEmpty(item.URLTemplate, item.APIURL),
"url_template": firstNonEmpty(item.URLTemplate, item.APIURL),
"thumbnail_url": item.ThumbnailURL,
"thumbnailUrl": item.ThumbnailURL,
"method": item.Method,
"proxy_mode": item.ProxyMode,
"proxyMode": item.ProxyMode,
"refresh_interval": item.CheckIntervalSec,
"refreshInterval": item.CheckIntervalSec,
"cacheSeconds": item.CacheSeconds,
"enabled": item.Enabled,
"clientVisible": item.ClientVisible,
"supported_formats": formats,
"supportedFormats": formats,
"downloadable": true,
"kind": inferMediaType(formats, item),
"mediaType": inferMediaType(formats, item),
"media_type": inferMediaType(formats, item),
"health": map[string]any{
"status": item.LastStatus,
"latency_ms": item.LastLatencyMS,
"latencyMs": item.LastLatencyMS,
"last_checked_at": item.LastCheckedAt,
"lastCheckedAt": item.LastCheckedAt,
"last_error": item.LastError,
"lastError": item.LastError,
"consecutiveFailure": item.ConsecutiveFailure,
"meta": parseHealthMeta(item.LastError),
},
@@ -274,21 +345,36 @@ func (s *Service) Endpoints(includeHidden bool) ([]map[string]any, error) {
var formats []string
_ = json.Unmarshal([]byte(item.SupportedFormats), &formats)
endpoint := map[string]any{
"id": item.SourceID,
"category": item.CategoryID,
"name": item.Name,
"method": item.Method,
"urlTemplate": firstNonEmpty(item.URLTemplate, item.APIURL),
"proxyMode": item.ProxyMode,
"clientVisible": item.ClientVisible,
"enabled": item.Enabled,
"cacheSeconds": item.CacheSeconds,
"supportedFormats": formats,
"id": item.SourceID,
"sourceId": item.SourceID,
"category": item.CategoryID,
"categoryId": item.CategoryID,
"categoryName": item.CategoryName,
"name": item.Name,
"description": item.Description,
"method": item.Method,
"apiUrl": item.APIURL,
"api_url": item.APIURL,
"urlTemplate": firstNonEmpty(item.URLTemplate, item.APIURL),
"url_template": firstNonEmpty(item.URLTemplate, item.APIURL),
"proxyMode": item.ProxyMode,
"proxy_mode": item.ProxyMode,
"clientVisible": item.ClientVisible,
"enabled": item.Enabled,
"cacheSeconds": item.CacheSeconds,
"supportedFormats": formats,
"supported_formats": formats,
"kind": inferMediaType(formats, item),
"mediaType": inferMediaType(formats, item),
"media_type": inferMediaType(formats, item),
"health": map[string]any{
"status": item.LastStatus,
"latencyMs": item.LastLatencyMS,
"latency_ms": item.LastLatencyMS,
"lastCheckedAt": item.LastCheckedAt,
"last_checked_at": item.LastCheckedAt,
"lastError": item.LastError,
"last_error": item.LastError,
"consecutiveFailure": item.ConsecutiveFailure,
"meta": parseHealthMeta(item.LastError),
},
@@ -826,6 +912,79 @@ func parseHealthMeta(message string) map[string]any {
return meta
}
func (c legacyCategory) entries() []legacySubcategory {
out := make([]legacySubcategory, 0, len(c.Subcategories)+len(c.Sources)+len(c.Endpoints))
out = append(out, c.Subcategories...)
out = append(out, c.Sources...)
out = append(out, c.Endpoints...)
return out
}
func firstNonEmptySlice(values ...[]string) []string {
for _, value := range values {
if len(value) > 0 {
return value
}
}
return nil
}
func firstPositive(values ...int) int {
for _, value := range values {
if value > 0 {
return value
}
}
return 0
}
func inferMediaType(formats []string, item db.Source) string {
for _, format := range formats {
if value := strings.ToLower(strings.TrimSpace(format)); value != "" {
switch value {
case "image", "video", "audio":
return value
}
}
}
if value := mediaTypeFromURL(mustParseURL(firstNonEmpty(item.URLTemplate, item.APIURL, item.ThumbnailURL))); value != "" {
return value
}
for _, format := range formats {
if value := mediaTypeFromFormat(format); value != "" {
return value
}
}
category := strings.ToLower(strings.TrimSpace(item.CategoryID + " " + item.CategoryName))
for _, candidate := range []string{"image", "video", "audio"} {
if strings.Contains(category, candidate) {
return candidate
}
}
return "media"
}
func mediaTypeFromFormat(value string) string {
switch strings.ToLower(strings.Trim(strings.TrimSpace(value), ".")) {
case "jpg", "jpeg", "png", "webp", "gif", "bmp", "tif", "tiff":
return "image"
case "mp4", "webm", "m3u8", "mkv", "mov", "m4v", "avi", "wmv":
return "video"
case "mp3", "wav", "flac", "aac", "m4a", "ogg", "wma":
return "audio"
default:
return ""
}
}
func mustParseURL(value string) *url.URL {
parsed, err := url.Parse(strings.TrimSpace(value))
if err != nil {
return nil
}
return parsed
}
func defaultString(value, fallback string) string {
if strings.TrimSpace(value) == "" {
return fallback
@@ -5,6 +5,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
@@ -229,6 +230,63 @@ func TestCheckOneResolvesTextMediaURL(t *testing.T) {
}
}
func TestImportLegacyMediaTypesRestoresClientVisibleSources(t *testing.T) {
cfg, store := testStore(t)
if err := os.MkdirAll(cfg.UpdatePublicDir, 0o755); err != nil {
t.Fatal(err)
}
mediaTypes := `{
"categories": [{
"categoryId": "image",
"name": "Images",
"sources": [{
"sourceId": "random-card",
"name": "Random Card",
"description": "Card description from source",
"apiUrl": "https://example.test/random-card",
"thumbnailUrl": "https://example.test/thumb.webp",
"supportedFormats": ["webp"],
"mediaType": "image"
}]
}]
}`
if err := os.WriteFile(filepath.Join(cfg.UpdatePublicDir, "media-types.json"), []byte(mediaTypes), 0o644); err != nil {
t.Fatal(err)
}
if _, err := store.UpsertSource(db.Source{
CategoryID: "hidden",
CategoryName: "Hidden",
SourceID: "hidden-source",
Name: "Hidden",
APIURL: "https://example.test/hidden",
Enabled: false,
ClientVisible: false,
EnabledSet: true,
ClientVisibleSet: true,
}); err != nil {
t.Fatal(err)
}
service := NewService(cfg, store)
if err := service.ImportLegacyMediaTypesIfEmpty(context.Background()); err != nil {
t.Fatal(err)
}
visible, err := store.CountClientVisibleSources()
if err != nil {
t.Fatal(err)
}
if visible != 1 {
t.Fatalf("visible source count = %d, want 1", visible)
}
catalog, err := service.Catalog(false)
if err != nil {
t.Fatal(err)
}
sub := catalog["categories"].([]map[string]any)[0]["subcategories"].([]map[string]any)[0]
if sub["description"] != "Card description from source" || sub["apiUrl"] != "https://example.test/random-card" || sub["mediaType"] != "image" {
t.Fatalf("catalog did not expose stable source fields: %#v", sub)
}
}
func testStore(t *testing.T) (*config.Config, *db.Store) {
t.Helper()
dir := t.TempDir()
@@ -24,6 +24,7 @@ func (r *router) handleAdminSources(w http.ResponseWriter, req *http.Request) {
writeError(w, http.StatusInternalServerError, "IMPORT_FAILED", err)
return
}
_ = r.sources.PublishLegacyMediaTypes(req.Context(), "admin")
writeJSON(w, http.StatusOK, map[string]any{"ok": true})
case req.Method == http.MethodPost && path == "/api/admin/sources/check":
job := r.sources.QueueCheckAll()
@@ -46,8 +47,8 @@ func (r *router) handleAdminSources(w http.ResponseWriter, req *http.Request) {
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "source": item})
case (req.Method == http.MethodPost || req.Method == http.MethodPut) && path == "/api/admin/sources":
var item db.Source
if err := json.NewDecoder(req.Body).Decode(&item); err != nil {
item, err := r.decodeAdminSource(req)
if err != nil {
writeError(w, http.StatusBadRequest, "INVALID_PAYLOAD", err)
return
}
@@ -72,3 +73,114 @@ func (r *router) handleAdminSources(w http.ResponseWriter, req *http.Request) {
http.NotFound(w, req)
}
}
type adminSourceRequest struct {
ID int64 `json:"id"`
CategoryID string `json:"categoryId"`
CategoryIDAlt string `json:"category_id"`
CategoryName string `json:"categoryName"`
CategoryNameAlt string `json:"category_name"`
SourceID string `json:"sourceId"`
SourceIDAlt string `json:"source_id"`
Name string `json:"name"`
Description string `json:"description"`
Method string `json:"method"`
APIURL string `json:"apiUrl"`
APIURLAlt string `json:"api_url"`
URLTemplate string `json:"urlTemplate"`
URLTemplateAlt string `json:"url_template"`
ThumbnailURL string `json:"thumbnailUrl"`
ThumbnailURLAlt string `json:"thumbnail_url"`
ProxyMode string `json:"proxyMode"`
ProxyModeAlt string `json:"proxy_mode"`
TimeoutMS int `json:"timeoutMs"`
TimeoutMSAlt int `json:"timeout_ms"`
RetryCount int `json:"retryCount"`
RetryCountAlt int `json:"retry_count"`
CacheSeconds int `json:"cacheSeconds"`
CacheSecondsAlt int `json:"cache_seconds"`
CheckIntervalSec int `json:"checkIntervalSec"`
CheckIntervalSecAlt int `json:"check_interval_sec"`
Enabled *bool `json:"enabled"`
ClientVisible *bool `json:"clientVisible"`
ClientVisibleAlt *bool `json:"client_visible"`
SupportedFormats []string `json:"supportedFormats"`
SupportedFormatsAlt []string `json:"supported_formats"`
LastStatus string `json:"lastStatus"`
LastStatusAlt string `json:"last_status"`
LastLatencyMS int `json:"lastLatencyMs"`
LastLatencyMSAlt int `json:"last_latency_ms"`
LastCheckedAt string `json:"lastCheckedAt"`
LastCheckedAtAlt string `json:"last_checked_at"`
LastError string `json:"lastError"`
LastErrorAlt string `json:"last_error"`
ConsecutiveFailure int `json:"consecutiveFailure"`
ConsecutiveFailAlt int `json:"consecutive_failure"`
}
func (r *router) decodeAdminSource(req *http.Request) (db.Source, error) {
var body adminSourceRequest
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
return db.Source{}, err
}
sourceID := firstNonEmpty(body.SourceID, body.SourceIDAlt)
var existing db.Source
hasExisting := false
if sourceID != "" {
if item, err := r.store.GetSourceBySourceID(sourceID); err == nil {
existing = item
hasExisting = true
}
}
item := existing
item.ID = body.ID
item.CategoryID = firstNonEmpty(body.CategoryID, body.CategoryIDAlt, item.CategoryID)
item.CategoryName = firstNonEmpty(body.CategoryName, body.CategoryNameAlt, item.CategoryName)
item.SourceID = firstNonEmpty(sourceID, item.SourceID)
item.Name = firstNonEmpty(body.Name, item.Name)
item.Description = body.Description
item.Method = firstNonEmpty(body.Method, item.Method)
item.APIURL = firstNonEmpty(body.APIURL, body.APIURLAlt, item.APIURL)
item.URLTemplate = firstNonEmpty(body.URLTemplate, body.URLTemplateAlt, item.URLTemplate, item.APIURL)
item.ThumbnailURL = firstNonEmpty(body.ThumbnailURL, body.ThumbnailURLAlt)
item.ProxyMode = firstNonEmpty(body.ProxyMode, body.ProxyModeAlt, item.ProxyMode)
item.TimeoutMS = firstPositive(body.TimeoutMS, body.TimeoutMSAlt, item.TimeoutMS)
item.RetryCount = firstPositive(body.RetryCount, body.RetryCountAlt, item.RetryCount)
item.CacheSeconds = firstPositive(body.CacheSeconds, body.CacheSecondsAlt, item.CacheSeconds)
item.CheckIntervalSec = firstPositive(body.CheckIntervalSec, body.CheckIntervalSecAlt, item.CheckIntervalSec)
item.LastStatus = firstNonEmpty(body.LastStatus, body.LastStatusAlt, item.LastStatus)
item.LastLatencyMS = firstPositive(body.LastLatencyMS, body.LastLatencyMSAlt, item.LastLatencyMS)
item.LastCheckedAt = firstNonEmpty(body.LastCheckedAt, body.LastCheckedAtAlt, item.LastCheckedAt)
item.LastError = firstNonEmpty(body.LastError, body.LastErrorAlt, item.LastError)
item.ConsecutiveFailure = firstPositive(body.ConsecutiveFailure, body.ConsecutiveFailAlt, item.ConsecutiveFailure)
if formats := firstNonEmptyStringSlice(body.SupportedFormats, body.SupportedFormatsAlt); len(formats) > 0 {
data, _ := json.Marshal(formats)
item.SupportedFormats = string(data)
}
if body.Enabled != nil {
item.Enabled = *body.Enabled
item.EnabledSet = true
} else if hasExisting {
item.Enabled = existing.Enabled
}
visible := body.ClientVisible
if visible == nil {
visible = body.ClientVisibleAlt
}
if visible != nil {
item.ClientVisible = *visible
item.ClientVisibleSet = true
} else if hasExisting {
item.ClientVisible = existing.ClientVisible
}
return item, nil
}
func firstNonEmptyStringSlice(values ...[]string) []string {
for _, value := range values {
if len(value) > 0 {
return value
}
}
return nil
}
@@ -6,6 +6,7 @@ import (
"net/http"
"path/filepath"
"strconv"
"strings"
"time"
"ymhut-box/server/unified-management/internal/config"
@@ -55,6 +56,40 @@ func (r *router) handleAdminDatabase(w http.ResponseWriter, req *http.Request) {
}
_ = r.store.InsertAudit(db.AuditLog{Actor: "admin", Type: "system.database.saved", Target: body.Provider, Message: "数据库配置已保存并热切换", IP: req.RemoteAddr, UserAgent: req.UserAgent()})
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "database": r.store.Status(), "config": config.SafeDatabase(r.cfg.BaseDir, r.cfg.Database)})
case req.Method == http.MethodPost && path == "/api/admin/database/sync/jobs":
var body struct {
Direction string `json:"direction"`
}
if err := json.NewDecoder(req.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "INVALID_PAYLOAD", err)
return
}
job, err := r.store.QueueDatabaseSync(body.Direction)
if err != nil {
writeError(w, http.StatusConflict, "DATABASE_SYNC_RUNNING", err)
return
}
_ = r.store.InsertAudit(db.AuditLog{Actor: "admin", Type: "database.sync.queued", Target: job.Direction, Message: "数据库同步任务已启动", IP: req.RemoteAddr, UserAgent: req.UserAgent()})
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "jobId": job.ID, "job": job})
case req.Method == http.MethodGet && path == "/api/admin/database/sync/jobs/latest":
job, err := r.store.LatestDatabaseSyncJob()
if err != nil {
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "job": nil})
return
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "job": job})
case req.Method == http.MethodGet && strings.HasPrefix(path, "/api/admin/database/sync/jobs/"):
id, err := strconv.ParseInt(strings.TrimPrefix(path, "/api/admin/database/sync/jobs/"), 10, 64)
if err != nil || id <= 0 {
writeError(w, http.StatusBadRequest, "INVALID_JOB_ID", errors.New("invalid sync job id"))
return
}
job, err := r.store.GetDatabaseSyncJob(id)
if err != nil {
writeError(w, http.StatusNotFound, "SYNC_JOB_NOT_FOUND", err)
return
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "job": job})
case req.Method == http.MethodPost && path == "/api/admin/database/import-sqlite":
result, err := r.store.ImportSQLiteToRemote()
if err != nil {
@@ -218,6 +253,22 @@ func (r *router) handleAdminSystem(w http.ResponseWriter, req *http.Request) {
return
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "items": page.Items, "page": page})
case "/api/admin/system/logs":
if req.Method != http.MethodGet {
writeError(w, http.StatusMethodNotAllowed, "METHOD_NOT_ALLOWED", errors.New("GET required"))
return
}
page, err := r.store.ListSystemLogsPage(db.SystemLogFilters{
Page: queryInt(req, "page", 1),
PerPage: queryInt(req, "perPage", 35),
Category: req.URL.Query().Get("category"),
Query: req.URL.Query().Get("q"),
})
if err != nil {
writeError(w, http.StatusInternalServerError, "SYSTEM_LOGS_FAILED", err)
return
}
writeJSON(w, http.StatusOK, map[string]any{"ok": true, "items": page.Items, "page": page})
case "/api/admin/system/mail/config":
r.handleMailConfig(w, req)
case "/api/admin/system/mail/test":
@@ -71,6 +71,12 @@ func (r *router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
r.auth.Require(http.HandlerFunc(r.handleChangePassword)).ServeHTTP(w, req)
case path == "/api/client/bootstrap":
r.handleClientBootstrap(w, req)
case path == "/favicon.ico" || path == "/admin/favicon.ico":
r.serveServerAsset(w, req, "favicon.ico")
case path == "/assets/favicon.ico":
r.serveServerAsset(w, req, "favicon.ico")
case path == "/assets/developer-avatar.png":
r.serveServerAsset(w, req, "developer-avatar.png")
case path == "/api/client/releases" || path == "/api/releases" || path == "/api/update-info":
writeJSON(w, http.StatusOK, r.releases.Manifest(req))
case path == "/api/client/sources":
@@ -447,6 +447,31 @@ func TestBuiltFrontendAssetsAreServed(t *testing.T) {
}
}
func TestServerBrandAssetsAreServed(t *testing.T) {
handler, cleanup := testRouter(t)
defer cleanup()
for _, item := range []struct {
Path string
ContentTypes []string
}{
{Path: "/favicon.ico", ContentTypes: []string{"image/x-icon", "image/vnd.microsoft.icon"}},
{Path: "/admin/favicon.ico", ContentTypes: []string{"image/x-icon", "image/vnd.microsoft.icon"}},
{Path: "/assets/favicon.ico", ContentTypes: []string{"image/x-icon", "image/vnd.microsoft.icon"}},
{Path: "/assets/developer-avatar.png", ContentTypes: []string{"image/png"}},
} {
req := httptest.NewRequest(http.MethodGet, item.Path, nil)
res := httptest.NewRecorder()
handler.ServeHTTP(res, req)
if res.Code != http.StatusOK {
t.Fatalf("%s returned %d: %s", item.Path, res.Code, res.Body.String())
}
if got := res.Header().Get("Content-Type"); !containsAny(got, item.ContentTypes) {
t.Fatalf("%s content type = %q, want one of %v", item.Path, got, item.ContentTypes)
}
}
}
func TestAdminSystemAndLegacyAdminPagesServeSPA(t *testing.T) {
handler, cleanup := testRouter(t)
defer cleanup()
@@ -720,9 +745,19 @@ func testRouter(t *testing.T) (http.Handler, func()) {
root := t.TempDir()
public := filepath.Join(root, "public")
noticeDir := filepath.Join(root, "update-notice")
assetDir := filepath.Join(root, "assets")
if err := os.MkdirAll(filepath.Join(public, "downloads"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(assetDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(assetDir, "favicon.ico"), []byte("ico"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(assetDir, "developer-avatar.png"), []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}, 0o644); err != nil {
t.Fatal(err)
}
adminDist := filepath.Join(root, "admin")
portalDist := filepath.Join(root, "portal")
for _, dir := range []string{filepath.Join(adminDist, "assets"), filepath.Join(portalDist, "assets")} {
@@ -17,10 +17,10 @@ type setupRouter struct {
}
type setupRequest struct {
Provider string `json:"provider"`
BaseURL string `json:"baseUrl"`
SQLitePath string `json:"sqlitePath"`
MySQLDSN string `json:"mysqlDsn"`
Provider string `json:"provider"`
BaseURL string `json:"baseUrl"`
SQLitePath string `json:"sqlitePath"`
MySQLDSN string `json:"mysqlDsn"`
MySQL config.MySQLInput `json:"mysql"`
}
@@ -33,6 +33,12 @@ func (r *setupRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
switch {
case path == "/" || path == "/setup":
r.serveSetup(w, req)
case path == "/favicon.ico" || path == "/setup/favicon.ico":
serveSetupServerAsset(w, req, r.cfg.BaseDir, "favicon.ico")
case path == "/assets/favicon.ico":
serveSetupServerAsset(w, req, r.cfg.BaseDir, "favicon.ico")
case path == "/assets/developer-avatar.png":
serveSetupServerAsset(w, req, r.cfg.BaseDir, "developer-avatar.png")
case strings.HasPrefix(path, "/setup/assets/"):
serveStaticAsset(w, req, r.cfg.SetupWebDir, "setup/dist", strings.TrimPrefix(path, "/setup/"))
case path == "/api/setup/status":
@@ -47,6 +47,14 @@ func serveStaticAsset(w http.ResponseWriter, req *http.Request, root, embedRoot,
http.NotFound(w, req)
}
func (r *router) serveServerAsset(w http.ResponseWriter, req *http.Request, assetPath string) {
serveStaticAsset(w, req, filepath.Join(r.cfg.BaseDir, "assets"), "", assetPath)
}
func serveSetupServerAsset(w http.ResponseWriter, req *http.Request, cfgRoot, assetPath string) {
serveStaticAsset(w, req, filepath.Join(cfgRoot, "assets"), "", assetPath)
}
func tryServeDiskFile(w http.ResponseWriter, req *http.Request, root, assetPath string) bool {
path := filepath.Join(root, filepath.FromSlash(assetPath))
resolved, err := filepath.Abs(path)