diff --git a/cmd/handlers.go b/cmd/handlers.go index 68604bc3..2efdc795 100644 --- a/cmd/handlers.go +++ b/cmd/handlers.go @@ -107,6 +107,7 @@ func initHTTPHandlers(e *echo.Echo, a *App) { g.GET("/api/settings", pm(a.GetSettings, "settings:get")) g.PUT("/api/settings", pm(a.UpdateSettings, "settings:manage")) + g.PUT("/api/settings/:key", pm(a.UpdateSettingsByKey, "settings:manage")) g.POST("/api/settings/smtp/test", pm(a.TestSMTPSettings, "settings:manage")) g.POST("/api/admin/reload", pm(a.ReloadApp, "settings:manage")) g.GET("/api/logs", pm(a.GetLogs, "settings:get")) diff --git a/cmd/init.go b/cmd/init.go index 24a1af37..613164ba 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -8,7 +8,9 @@ import ( "errors" "fmt" "html/template" + "io" "log" + "log/slog" "maps" "net/http" "os" @@ -941,27 +943,49 @@ func initCaptcha() *captcha.Captcha { return captcha.New(opt) } -// initCron initializes the cron job for refreshing slow query cache. -func initCron(co *core.Core) { - intval := ko.String("app.cache_slow_queries_interval") - if intval == "" { - lo.Println("error: invalid cron interval string") - return +// initCron initializes cron jobs for slow query cache refresh and database vacuum. +func initCron(co *core.Core, db *sqlx.DB) { + c := cron.New(cron.WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil)))) + + // Slow query cache cron job. + if ko.Bool("app.cache_slow_queries") { + intval := ko.String("app.cache_slow_queries_interval") + if intval == "" { + lo.Println("error: invalid cron interval string for slow query cache") + } else { + _, err := c.Add(intval, func() { + lo.Println("refreshing slow query cache") + _ = co.RefreshMatViews(true) + lo.Println("done refreshing slow query cache") + }) + if err != nil { + lo.Printf("error initializing slow cache query cron: %v", err) + } else { + lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[len(c.Entries())-1].Next) + } + } } - c := cron.New() - _, err := c.Add(intval, func() { - lo.Println("refreshing slow query cache") - _ = co.RefreshMatViews(true) - lo.Println("done refreshing slow query cache") - }) - if err != nil { - lo.Printf("error initializing slow cache query cron: %v", err) - return + // Database vacuum cron job. + if ko.Bool("maintenance.db.vacuum") { + intval := ko.String("maintenance.db.vacuum_cron_interval") + if intval == "" { + lo.Println("error: invalid cron interval string for database vacuum") + } else { + _, err := c.Add(intval, func() { + RunDBVacuum(db, lo) + }) + if err != nil { + lo.Printf("error initializing database vacuum cron: %v", err) + } else { + lo.Printf("database VACUUM cron enabled at interval: %s", intval) + } + } } - c.Start() - lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[0].Next) + if len(c.Entries()) > 0 { + c.Start() + } } // awaitReload waits for a SIGHUP signal to reload the app. Every setting change on the UI causes a reload. diff --git a/cmd/main.go b/cmd/main.go index c10f658d..082303a2 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -232,9 +232,7 @@ func main() { } // Start cronjobs. - if ko.Bool("app.cache_slow_queries") { - initCron(core) - } + initCron(core, db) // Start the campaign manager workers. The campaign batches (fetch from DB, push out // messages) get processed at the specified interval. diff --git a/cmd/maintenance.go b/cmd/maintenance.go index 5bdac8c0..2a8d8395 100644 --- a/cmd/maintenance.go +++ b/cmd/maintenance.go @@ -1,9 +1,11 @@ package main import ( + "log" "net/http" "time" + "github.com/jmoiron/sqlx" "github.com/labstack/echo/v4" ) @@ -81,3 +83,14 @@ func (a *App) GCCampaignAnalytics(c echo.Context) error { return c.JSON(http.StatusOK, okResp{true}) } + +// RunDBVacuum runs a full VACUUM on the PostgreSQL database. +// VACUUM reclaims storage occupied by dead tuples and updates planner statistics. +func RunDBVacuum(db *sqlx.DB, lo *log.Logger) { + lo.Println("running database VACUUM ANALYZE") + if _, err := db.Exec("VACUUM ANALYZE"); err != nil { + lo.Printf("error running VACUUM ANALYZE: %v", err) + return + } + lo.Println("finished database VACUUM ANALYZE") +} diff --git a/cmd/settings.go b/cmd/settings.go index ccec8bc9..12ecf2f7 100644 --- a/cmd/settings.go +++ b/cmd/settings.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "encoding/json" "io" "net/http" "net/url" @@ -15,7 +16,7 @@ import ( "github.com/gdgvda/cron" "github.com/gofrs/uuid/v5" "github.com/jmoiron/sqlx/types" - "github.com/knadh/koanf/parsers/json" + koanfjson "github.com/knadh/koanf/parsers/json" "github.com/knadh/koanf/providers/rawbytes" "github.com/knadh/koanf/v2" "github.com/knadh/listmonk/internal/auth" @@ -290,6 +291,33 @@ func (a *App) UpdateSettings(c echo.Context) error { return err } + return a.handleSettingsRestart(c) +} + +// UpdateSettingsByKey updates a single setting key-value in the DB. +func (a *App) UpdateSettingsByKey(c echo.Context) error { + key := c.Param("key") + if key == "" { + return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("globals.messages.invalidData")) + } + + // Read the raw JSON body as the value. + var b json.RawMessage + if err := c.Bind(&b); err != nil { + return err + } + + // Update the value in the DB. + if err := a.core.UpdateSettingsByKey(key, b); err != nil { + return err + } + + return a.handleSettingsRestart(c) +} + +// handleSettingsRestart checks for running campaigns and either triggers an +// immediate app restart or marks the app as needing a restart. +func (a *App) handleSettingsRestart(c echo.Context) error { // If there are any active campaigns, don't do an auto reload and // warn the user on the frontend. if a.manager.HasRunningCampaigns() { @@ -302,7 +330,7 @@ func (a *App) UpdateSettings(c echo.Context) error { }{true}}) } - // No running campaigns. Reload the a. + // No running campaigns. Reload the app. go func() { <-time.After(time.Millisecond * 500) a.chReload <- syscall.SIGHUP @@ -327,7 +355,7 @@ func (a *App) TestSMTPSettings(c echo.Context) error { // Load the JSON into koanf to parse SMTP settings properly including timestrings. ko := koanf.New(".") - if err := ko.Load(rawbytes.Provider(reqBody), json.Parser()); err != nil { + if err := ko.Load(rawbytes.Provider(reqBody), koanfjson.Parser()); err != nil { a.log.Printf("error unmarshalling SMTP test request: %v", err) return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.internalError")) } diff --git a/frontend/src/api/index.js b/frontend/src/api/index.js index 842e4e78..cac1af8f 100644 --- a/frontend/src/api/index.js +++ b/frontend/src/api/index.js @@ -431,6 +431,12 @@ export const updateSettings = async (data) => http.put( { loading: models.settings }, ); +export const updateSettingsByKey = async (key, data) => http.put( + `/api/settings/${key}`, + data, + { loading: models.settings }, +); + export const testSMTP = async (data) => http.post( '/api/settings/smtp/test', data, diff --git a/frontend/src/main.js b/frontend/src/main.js index 0b914c5c..e36c137f 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -101,6 +101,31 @@ const v = new Vue({ loadConfig() { initConfig(); }, + + // awaitRestart handles app restart polling after settings changes. + // Shows a toast and polls until the backend is back up. + // Returns a promise that resolves with { needsRestart: boolean }. + awaitRestart(response) { + return new Promise((resolve) => { + // If there are running campaigns, app won't auto restart. + if (response && typeof response === 'object' && response.needsRestart) { + this.loadConfig(); + resolve({ needsRestart: true }); + return; + } + + Vue.prototype.$utils.toast(i18n.t('settings.messengers.messageSaved')); + + // Poll until backend is back up. + const pollId = setInterval(() => { + api.getHealth().then(() => { + clearInterval(pollId); + this.loadConfig(); + resolve({ needsRestart: false }); + }); + }, 1000); + }); + }, }, mounted() { diff --git a/frontend/src/views/Maintenance.vue b/frontend/src/views/Maintenance.vue index 0363f126..fdeed2a2 100644 --- a/frontend/src/views/Maintenance.vue +++ b/frontend/src/views/Maintenance.vue @@ -107,6 +107,39 @@ + +
+

+ {{ $t('maintenance.database.title') }} +


+
Vacuum
+

+ {{ $t('maintenance.database.vacuumHelp') }} +

+
+
+
+ + + +
+
+ + + +
+
+
+
+ + {{ $t('globals.buttons.save') }} + +
+
+ + + @@ -121,14 +154,23 @@ export default Vue.extend({ data() { return { + isLoading: false, subscriberType: 'orphan', analyticsType: 'all', subscriptionType: 'optin', analyticsDate: dayjs().subtract(7, 'day').toDate(), subscriptionDate: dayjs().subtract(7, 'day').toDate(), + dbSettings: { + vacuum: false, + vacuum_cron_interval: '0 2 * * *', + }, }; }, + mounted() { + this.loadDBSettings(); + }, + methods: { formatDateTime(s) { return dayjs(s).format('YYYY-MM-DD'); @@ -173,6 +215,21 @@ export default Vue.extend({ }, ); }, + + loadDBSettings() { + this.$api.getSettings().then((data) => { + if (data['maintenance.db'] !== undefined) { + this.dbSettings = { ...data['maintenance.db'] }; + } + }); + }, + + async onUpdateDBSettings() { + this.isLoading = true; + const data = await this.$api.updateSettingsByKey('maintenance.db', this.dbSettings); + await this.$root.awaitRestart(data); + this.isLoading = false; + }, }, computed: { diff --git a/frontend/src/views/Settings.vue b/frontend/src/views/Settings.vue index 5a33de55..d476055e 100644 --- a/frontend/src/views/Settings.vue +++ b/frontend/src/views/Settings.vue @@ -106,7 +106,7 @@ export default Vue.extend({ }, methods: { - onSubmit() { + async onSubmit() { const form = JSON.parse(JSON.stringify(this.form)); // SMTP boxes. @@ -197,30 +197,13 @@ export default Vue.extend({ form['privacy.domain_allowlist'] = form['privacy.domain_allowlist'].split('\n').map((v) => v.trim().toLowerCase()).filter((v) => v !== ''); this.isLoading = true; - this.$api.updateSettings(form).then((data) => { - if (typeof data === 'object' && data !== null && data.needsRestart) { - // There are running campaigns and the app didn't auto restart. - // The UI will show a warning. - this.$root.loadConfig(); - this.getSettings(); - this.isLoading = false; - return; - } - - this.$utils.toast(this.$t('settings.messengers.messageSaved')); - - // Poll until there's a 200 response, waiting for the app - // to restart and come back up. - const pollId = setInterval(() => { - this.$api.getHealth().then(() => { - clearInterval(pollId); - this.$root.loadConfig(); - this.getSettings(); - }); - }, 1000); - }, () => { + try { + const data = await this.$api.updateSettings(form); + await this.$root.awaitRestart(data); + this.getSettings(); + } finally { this.isLoading = false; - }); + } return false; }, diff --git a/i18n/en.json b/i18n/en.json index e6c4bd50..c6ed4aef 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -686,5 +686,7 @@ "users.totpSecret": "Secret key", "users.invalidPassword": "Invalid password", "lists.archived": "Archived", - "lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists." + "lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists.", + "maintenance.database.title": "Database", + "maintenance.database.vacuumHelp": "PostgreSQL VACUUM ANALYZE reclaims storage used by deleted rows and significantly speeds up database performance on large databases. IMPORTANT: For large databases, this is a slow, blocking operation. Schedule to run this during off-peak hours." } diff --git a/internal/core/settings.go b/internal/core/settings.go index c3dec64c..2d277e46 100644 --- a/internal/core/settings.go +++ b/internal/core/settings.go @@ -48,3 +48,13 @@ func (c *Core) UpdateSettings(s models.Settings) error { return nil } + +// UpdateSettingsByKey updates a single setting by key. +func (c *Core) UpdateSettingsByKey(key string, value json.RawMessage) error { + if _, err := c.q.UpdateSettingsByKey.Exec(key, value); err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, + c.i18n.Ts("globals.messages.errorUpdating", "name", "{globals.terms.settings}", "error", pqErrMsg(err))) + } + + return nil +} diff --git a/internal/migrations/v5.1.0.go b/internal/migrations/v5.1.0.go index 5b93e9e0..033622c1 100644 --- a/internal/migrations/v5.1.0.go +++ b/internal/migrations/v5.1.0.go @@ -50,5 +50,15 @@ func V5_1_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf, lo *log.Logger return err } + // Add maintenance.db setting if not present. + _, err = db.Exec(` + INSERT INTO settings (key, value, updated_at) + VALUES ('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}', NOW()) + ON CONFLICT (key) DO NOTHING + `) + if err != nil { + return err + } + return nil } diff --git a/models/queries.go b/models/queries.go index 6922b689..0703b6c3 100644 --- a/models/queries.go +++ b/models/queries.go @@ -104,8 +104,9 @@ type Queries struct { CreateLink *sqlx.Stmt `query:"create-link"` RegisterLinkClick *sqlx.Stmt `query:"register-link-click"` - GetSettings *sqlx.Stmt `query:"get-settings"` - UpdateSettings *sqlx.Stmt `query:"update-settings"` + GetSettings *sqlx.Stmt `query:"get-settings"` + UpdateSettings *sqlx.Stmt `query:"update-settings"` + UpdateSettingsByKey *sqlx.Stmt `query:"update-settings-by-key"` // GetStats *sqlx.Stmt `query:"get-stats"` RecordBounce *sqlx.Stmt `query:"record-bounce"` diff --git a/models/settings.go b/models/settings.go index 503b5fef..0051cb17 100644 --- a/models/settings.go +++ b/models/settings.go @@ -143,6 +143,11 @@ type Settings struct { ScanInterval string `json:"scan_interval"` } `json:"bounce.mailboxes"` + MaintenanceDB struct { + Vacuum bool `json:"vacuum"` + VacuumInterval string `json:"vacuum_cron_interval"` + } `json:"maintenance.db"` + AdminCustomCSS string `json:"appearance.admin.custom_css"` AdminCustomJS string `json:"appearance.admin.custom_js"` PublicCustomCSS string `json:"appearance.public.custom_css"` diff --git a/queries/misc.sql b/queries/misc.sql index 72f18f62..5b2e0bbe 100644 --- a/queries/misc.sql +++ b/queries/misc.sql @@ -12,6 +12,9 @@ UPDATE settings AS s SET value = c.value -- For each key in the incoming JSON map, update the row with the key and its value. FROM(SELECT * FROM JSONB_EACH($1)) AS c(key, value) WHERE s.key = c.key; +-- name: update-settings-by-key +UPDATE settings SET value = $2, updated_at = NOW() WHERE key = $1; + -- name: get-db-info SELECT JSON_BUILD_OBJECT('version', (SELECT VERSION()), 'size_mb', (SELECT ROUND(pg_database_size((SELECT CURRENT_DATABASE()))/(1024^2)))) AS info; diff --git a/schema.sql b/schema.sql index 09735051..80a0cd81 100644 --- a/schema.sql +++ b/schema.sql @@ -292,7 +292,8 @@ INSERT INTO settings (key, value) VALUES ('appearance.admin.custom_css', '""'), ('appearance.admin.custom_js', '""'), ('appearance.public.custom_css', '""'), - ('appearance.public.custom_js', '""'); + ('appearance.public.custom_js', '""'), + ('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}'); -- bounces DROP TABLE IF EXISTS bounces CASCADE;