eessppeelllloo/listmonk
1
0
Fork 0
mirror of https://github.com/knadh/listmonk.git synced 2025-12-05 16:00:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add cron-based VACUUM ANALYZE support for DB maintenance.

Browse Source 
- Add a new vacuum setting option on the UI in Admin -> Settings -> Maintenance.
- Also refactor frontend (lock-and-wait-for-restart) login on settings into
  the global vue instance so that it can be reused across contexts.
  Settings.vue and Maintenance.vue both now use it to wait for the backend
  to restart.
This commit is contained in:
Kailash Nadh
2025-11-28 20:01:11 +05:30
parent 67ad4d54ce
commit 570bb46d75
16 changed files with 218 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/handlers.go
View File

@@ -107,6 +107,7 @@ func initHTTPHandlers(e *echo.Echo, a *App) {
g.GET("/api/settings", pm(a.GetSettings, "settings:get"))
g.PUT("/api/settings", pm(a.UpdateSettings, "settings:manage"))
g.PUT("/api/settings/:key", pm(a.UpdateSettingsByKey, "settings:manage"))
g.POST("/api/settings/smtp/test", pm(a.TestSMTPSettings, "settings:manage"))
g.POST("/api/admin/reload", pm(a.ReloadApp, "settings:manage"))
g.GET("/api/logs", pm(a.GetLogs, "settings:get"))

58
cmd/init.go
View File

@@ -8,7 +8,9 @@ import (
"errors"
"fmt"
"html/template"
"io"
"log"
"log/slog"
"maps"
"net/http"
"os"
@@ -941,27 +943,49 @@ func initCaptcha() *captcha.Captcha {
return captcha.New(opt)
}
// initCron initializes the cron job for refreshing slow query cache.
func initCron(co *core.Core) {
intval := ko.String("app.cache_slow_queries_interval")
if intval == "" {
lo.Println("error: invalid cron interval string")
return
// initCron initializes cron jobs for slow query cache refresh and database vacuum.
func initCron(co *core.Core, db *sqlx.DB) {
c := cron.New(cron.WithLogger(slog.New(slog.NewTextHandler(io.Discard, nil))))
// Slow query cache cron job.
if ko.Bool("app.cache_slow_queries") {
intval := ko.String("app.cache_slow_queries_interval")
if intval == "" {
lo.Println("error: invalid cron interval string for slow query cache")
} else {
_, err := c.Add(intval, func() {
lo.Println("refreshing slow query cache")
_ = co.RefreshMatViews(true)
lo.Println("done refreshing slow query cache")
})
if err != nil {
lo.Printf("error initializing slow cache query cron: %v", err)
} else {
lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[len(c.Entries())-1].Next)
}
}
}
c := cron.New()
_, err := c.Add(intval, func() {
lo.Println("refreshing slow query cache")
_ = co.RefreshMatViews(true)
lo.Println("done refreshing slow query cache")
})
if err != nil {
lo.Printf("error initializing slow cache query cron: %v", err)
return
// Database vacuum cron job.
if ko.Bool("maintenance.db.vacuum") {
intval := ko.String("maintenance.db.vacuum_cron_interval")
if intval == "" {
lo.Println("error: invalid cron interval string for database vacuum")
} else {
_, err := c.Add(intval, func() {
RunDBVacuum(db, lo)
})
if err != nil {
lo.Printf("error initializing database vacuum cron: %v", err)
} else {
lo.Printf("database VACUUM cron enabled at interval: %s", intval)
}
}
}
c.Start()
lo.Printf("IMPORTANT: database slow query caching is enabled. Aggregate numbers and stats will not be realtime. Next refresh at: %v", c.Entries()[0].Next)
if len(c.Entries()) > 0 {
c.Start()
}
}
// awaitReload waits for a SIGHUP signal to reload the app. Every setting change on the UI causes a reload.

4
cmd/main.go
View File

@@ -232,9 +232,7 @@ func main() {
}
// Start cronjobs.
if ko.Bool("app.cache_slow_queries") {
initCron(core)
}
initCron(core, db)
// Start the campaign manager workers. The campaign batches (fetch from DB, push out
// messages) get processed at the specified interval.

13
cmd/maintenance.go
View File

@@ -1,9 +1,11 @@
package main
import (
"log"
"net/http"
"time"
"github.com/jmoiron/sqlx"
"github.com/labstack/echo/v4"
)
@@ -81,3 +83,14 @@ func (a *App) GCCampaignAnalytics(c echo.Context) error {
return c.JSON(http.StatusOK, okResp{true})
}
// RunDBVacuum runs a full VACUUM on the PostgreSQL database.
// VACUUM reclaims storage occupied by dead tuples and updates planner statistics.
func RunDBVacuum(db *sqlx.DB, lo *log.Logger) {
lo.Println("running database VACUUM ANALYZE")
if _, err := db.Exec("VACUUM ANALYZE"); err != nil {
lo.Printf("error running VACUUM ANALYZE: %v", err)
return
}
lo.Println("finished database VACUUM ANALYZE")
}

34
cmd/settings.go
View File

@@ -2,6 +2,7 @@ package main
import (
"bytes"
"encoding/json"
"io"
"net/http"
"net/url"
@@ -15,7 +16,7 @@ import (
"github.com/gdgvda/cron"
"github.com/gofrs/uuid/v5"
"github.com/jmoiron/sqlx/types"
"github.com/knadh/koanf/parsers/json"
koanfjson "github.com/knadh/koanf/parsers/json"
"github.com/knadh/koanf/providers/rawbytes"
"github.com/knadh/koanf/v2"
"github.com/knadh/listmonk/internal/auth"
@@ -290,6 +291,33 @@ func (a *App) UpdateSettings(c echo.Context) error {
return err
}
return a.handleSettingsRestart(c)
}
// UpdateSettingsByKey updates a single setting key-value in the DB.
func (a *App) UpdateSettingsByKey(c echo.Context) error {
key := c.Param("key")
if key == "" {
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.T("globals.messages.invalidData"))
}
// Read the raw JSON body as the value.
var b json.RawMessage
if err := c.Bind(&b); err != nil {
return err
}
// Update the value in the DB.
if err := a.core.UpdateSettingsByKey(key, b); err != nil {
return err
}
return a.handleSettingsRestart(c)
}
// handleSettingsRestart checks for running campaigns and either triggers an
// immediate app restart or marks the app as needing a restart.
func (a *App) handleSettingsRestart(c echo.Context) error {
// If there are any active campaigns, don't do an auto reload and
// warn the user on the frontend.
if a.manager.HasRunningCampaigns() {
@@ -302,7 +330,7 @@ func (a *App) UpdateSettings(c echo.Context) error {
}{true}})
}
// No running campaigns. Reload the a.
// No running campaigns. Reload the app.
go func() {
<-time.After(time.Millisecond * 500)
a.chReload <- syscall.SIGHUP
@@ -327,7 +355,7 @@ func (a *App) TestSMTPSettings(c echo.Context) error {
// Load the JSON into koanf to parse SMTP settings properly including timestrings.
ko := koanf.New(".")
if err := ko.Load(rawbytes.Provider(reqBody), json.Parser()); err != nil {
if err := ko.Load(rawbytes.Provider(reqBody), koanfjson.Parser()); err != nil {
a.log.Printf("error unmarshalling SMTP test request: %v", err)
return echo.NewHTTPError(http.StatusBadRequest, a.i18n.Ts("globals.messages.internalError"))
}

6
frontend/src/api/index.js
View File

@@ -431,6 +431,12 @@ export const updateSettings = async (data) => http.put(
{ loading: models.settings },
);
export const updateSettingsByKey = async (key, data) => http.put(
`/api/settings/${key}`,
data,
{ loading: models.settings },
);
export const testSMTP = async (data) => http.post(
'/api/settings/smtp/test',
data,

25
frontend/src/main.js
View File

@@ -101,6 +101,31 @@ const v = new Vue({
loadConfig() {
initConfig();
},
// awaitRestart handles app restart polling after settings changes.
// Shows a toast and polls until the backend is back up.
// Returns a promise that resolves with { needsRestart: boolean }.
awaitRestart(response) {
return new Promise((resolve) => {
// If there are running campaigns, app won't auto restart.
if (response && typeof response === 'object' && response.needsRestart) {
this.loadConfig();
resolve({ needsRestart: true });
return;
}
Vue.prototype.$utils.toast(i18n.t('settings.messengers.messageSaved'));
// Poll until backend is back up.
const pollId = setInterval(() => {
api.getHealth().then(() => {
clearInterval(pollId);
this.loadConfig();
resolve({ needsRestart: false });
});
}, 1000);
});
},
},
mounted() {

57
frontend/src/views/Maintenance.vue
View File

@@ -107,6 +107,39 @@
</div>
</div>
</div><!-- analytics -->
<form @submit.prevent="onUpdateDBSettings" class="box mt-6">
<h4 class="is-size-4">
{{ $t('maintenance.database.title') }}
</h4><br />
<h5 class="is-size-5">Vacuum</h5>
<p class="has-text-grey is-size-7">
{{ $t('maintenance.database.vacuumHelp') }}
</p>
<br />
<div class="columns">
<div class="column is-2">
<b-field :label="$t('globals.buttons.enabled')">
<b-switch v-model="dbSettings.vacuum" />
</b-field>
</div>
<div class="column is-4" :class="{ disabled: !dbSettings.vacuum }">
<b-field :label="$t('settings.maintenance.cron')">
<b-input v-model="dbSettings.vacuum_cron_interval" placeholder="0 2 * * *" :disabled="!dbSettings.vacuum"
pattern="((\*|[0-9,\-\/]+)\s+){4}(\*|[0-9,\-\/]+)" />
</b-field>
</div>
<div class="column is-3" />
<div class="column is-3">
<br />
<b-button type="is-primary" native-type="submit" :loading="loading.settings" expanded>
{{ $t('globals.buttons.save') }}
</b-button>
</div>
</div>
</form><!-- database -->
<b-loading :is-full-page="true" v-if="isLoading" active />
</section>
</template>
@@ -121,14 +154,23 @@ export default Vue.extend({
data() {
return {
isLoading: false,
subscriberType: 'orphan',
analyticsType: 'all',
subscriptionType: 'optin',
analyticsDate: dayjs().subtract(7, 'day').toDate(),
subscriptionDate: dayjs().subtract(7, 'day').toDate(),
dbSettings: {
vacuum: false,
vacuum_cron_interval: '0 2 * * *',
},
};
},
mounted() {
this.loadDBSettings();
},
methods: {
formatDateTime(s) {
return dayjs(s).format('YYYY-MM-DD');
@@ -173,6 +215,21 @@ export default Vue.extend({
},
);
},
loadDBSettings() {
this.$api.getSettings().then((data) => {
if (data['maintenance.db'] !== undefined) {
this.dbSettings = { ...data['maintenance.db'] };
}
});
},
async onUpdateDBSettings() {
this.isLoading = true;
const data = await this.$api.updateSettingsByKey('maintenance.db', this.dbSettings);
await this.$root.awaitRestart(data);
this.isLoading = false;
},
},
computed: {

31
frontend/src/views/Settings.vue
View File

@@ -106,7 +106,7 @@ export default Vue.extend({
},
methods: {
onSubmit() {
async onSubmit() {
const form = JSON.parse(JSON.stringify(this.form));
// SMTP boxes.
@@ -197,30 +197,13 @@ export default Vue.extend({
form['privacy.domain_allowlist'] = form['privacy.domain_allowlist'].split('\n').map((v) => v.trim().toLowerCase()).filter((v) => v !== '');
this.isLoading = true;
this.$api.updateSettings(form).then((data) => {
if (typeof data === 'object' && data !== null && data.needsRestart) {
// There are running campaigns and the app didn't auto restart.
// The UI will show a warning.
this.$root.loadConfig();
this.getSettings();
this.isLoading = false;
return;
}
this.$utils.toast(this.$t('settings.messengers.messageSaved'));
// Poll until there's a 200 response, waiting for the app
// to restart and come back up.
const pollId = setInterval(() => {
this.$api.getHealth().then(() => {
clearInterval(pollId);
this.$root.loadConfig();
this.getSettings();
});
}, 1000);
}, () => {
try {
const data = await this.$api.updateSettings(form);
await this.$root.awaitRestart(data);
this.getSettings();
} finally {
this.isLoading = false;
});
}
return false;
},

4
i18n/en.json
View File

@@ -686,5 +686,7 @@
"users.totpSecret": "Secret key",
"users.invalidPassword": "Invalid password",
"lists.archived": "Archived",
"lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists."
"lists.archivedHelp": "Archiving hides the lists from lists page, campaigns, and public forms. It can be unarchived anytime. It is useful for hiding old and rarely used lists.",
"maintenance.database.title": "Database",
"maintenance.database.vacuumHelp": "PostgreSQL VACUUM ANALYZE reclaims storage used by deleted rows and significantly speeds up database performance on large databases. IMPORTANT: For large databases, this is a slow, blocking operation. Schedule to run this during off-peak hours."
}

10
internal/core/settings.go
View File

@@ -48,3 +48,13 @@ func (c *Core) UpdateSettings(s models.Settings) error {
return nil
}
// UpdateSettingsByKey updates a single setting by key.
func (c *Core) UpdateSettingsByKey(key string, value json.RawMessage) error {
if _, err := c.q.UpdateSettingsByKey.Exec(key, value); err != nil {
return echo.NewHTTPError(http.StatusInternalServerError,
c.i18n.Ts("globals.messages.errorUpdating", "name", "{globals.terms.settings}", "error", pqErrMsg(err)))
}
return nil
}

10
internal/migrations/v5.1.0.go
View File

@@ -50,5 +50,15 @@ func V5_1_0(db *sqlx.DB, fs stuffbin.FileSystem, ko *koanf.Koanf, lo *log.Logger
return err
}
// Add maintenance.db setting if not present.
_, err = db.Exec(`
INSERT INTO settings (key, value, updated_at)
VALUES ('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}', NOW())
ON CONFLICT (key) DO NOTHING
`)
if err != nil {
return err
}
return nil
}

5
models/queries.go
View File

@@ -104,8 +104,9 @@ type Queries struct {
CreateLink *sqlx.Stmt `query:"create-link"`
RegisterLinkClick *sqlx.Stmt `query:"register-link-click"`
GetSettings *sqlx.Stmt `query:"get-settings"`
UpdateSettings *sqlx.Stmt `query:"update-settings"`
GetSettings *sqlx.Stmt `query:"get-settings"`
UpdateSettings *sqlx.Stmt `query:"update-settings"`
UpdateSettingsByKey *sqlx.Stmt `query:"update-settings-by-key"`
// GetStats *sqlx.Stmt `query:"get-stats"`
RecordBounce *sqlx.Stmt `query:"record-bounce"`

5
models/settings.go
View File

@@ -143,6 +143,11 @@ type Settings struct {
ScanInterval string `json:"scan_interval"`
} `json:"bounce.mailboxes"`
MaintenanceDB struct {
Vacuum bool `json:"vacuum"`
VacuumInterval string `json:"vacuum_cron_interval"`
} `json:"maintenance.db"`
AdminCustomCSS string `json:"appearance.admin.custom_css"`
AdminCustomJS string `json:"appearance.admin.custom_js"`
PublicCustomCSS string `json:"appearance.public.custom_css"`

3
queries/misc.sql
View File

@@ -12,6 +12,9 @@ UPDATE settings AS s SET value = c.value
-- For each key in the incoming JSON map, update the row with the key and its value.
FROM(SELECT * FROM JSONB_EACH($1)) AS c(key, value) WHERE s.key = c.key;
-- name: update-settings-by-key
UPDATE settings SET value = $2, updated_at = NOW() WHERE key = $1;
-- name: get-db-info
SELECT JSON_BUILD_OBJECT('version', (SELECT VERSION()),
'size_mb', (SELECT ROUND(pg_database_size((SELECT CURRENT_DATABASE()))/(1024^2)))) AS info;

3
schema.sql
View File

@@ -292,7 +292,8 @@ INSERT INTO settings (key, value) VALUES
('appearance.admin.custom_css', '""'),
('appearance.admin.custom_js', '""'),
('appearance.public.custom_css', '""'),
('appearance.public.custom_js', '""');
('appearance.public.custom_js', '""'),
('maintenance.db', '{"vacuum": false, "vacuum_cron_interval": "0 2 * * *"}');
-- bounces
DROP TABLE IF EXISTS bounces CASCADE;