Skip to content

Commit

Permalink
Add 'automated_backups' & 'force_backup' flags to gain more control o…
Browse files Browse the repository at this point in the history 
…ver when backups are written
  • Loading branch information
@carlosabalde
carlosabalde committed Jun 4, 2024
1 parent c972a32 commit 368cace
Show file tree
Hide file tree
Showing 8 changed files with 167 additions and 65 deletions.
9 changes: 6 additions & 3 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import cfg;
Object file(
STRING location,
STRING backup="",
BOOL automated_backups=1,
INT period=60,
BOOL ignore_load_failures=1,
INT curl_connection_timeout=0,
Expand All @@ -50,7 +51,7 @@ import cfg;
ENUM { ini, json } format="ini",
STRING name_delimiter=":",
STRING value_delimiter=";")
Method BOOL .reload()
Method BOOL .reload(BOOL force_backup=0)
Method STRING .dump(BOOL stream=0, STRING prefix="")
Method VOID .inspect()

Expand All @@ -64,6 +65,7 @@ import cfg;
Object rules(
STRING location,
STRING backup="",
BOOL automated_backups=1,
INT period=60,
BOOL ignore_load_failures=1,
INT curl_connection_timeout=0,
Expand All @@ -73,7 +75,7 @@ import cfg;
STRING curl_ssl_cafile="",
STRING curl_ssl_capath="",
STRING curl_proxy="")
Method BOOL .reload()
Method BOOL .reload(BOOL force_backup=0)
Method VOID .inspect()

Method STRING .get(STRING value, STRING fallback="")
Expand All @@ -85,6 +87,7 @@ import cfg;
Object script(
STRING location="",
STRING backup="",
BOOL automated_backups=1,
INT period=60,
BOOL ignore_load_failures=1,
ENUM { lua, javascript } type="lua",
Expand All @@ -105,7 +108,7 @@ import cfg;
STRING curl_ssl_cafile="",
STRING curl_ssl_capath="",
STRING curl_proxy="")
Method BOOL .reload()
Method BOOL .reload(BOOL force_backup=0)
Method VOID .inspect()

Method VOID .init(STRING code="")
Expand Down
65 changes: 37 additions & 28 deletions src/remote.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ static char *read_url(VRT_CTX, remote_t *remote);

remote_t *
new_remote(
const char *location, const char *backup, unsigned period, unsigned curl_connection_timeout,
const char *location, const char *backup, unsigned automated_backups,
unsigned period, unsigned curl_connection_timeout,
unsigned curl_transfer_timeout, unsigned curl_ssl_verify_peer,
unsigned curl_ssl_verify_host, const char *curl_ssl_cafile,
const char *curl_ssl_capath, const char *curl_proxy)
Expand All @@ -68,6 +69,7 @@ new_remote(
} else {
result->backup = NULL;
}
result->automated_backups = automated_backups;
result->period = period;
result->curl.connection_timeout = curl_connection_timeout;
result->curl.transfer_timeout = curl_transfer_timeout;
Expand Down Expand Up @@ -109,6 +111,7 @@ free_remote(remote_t *remote)
FREE_STRING(location.raw);
FREE_STRING(location.parsed);
remote->backup = NULL;
remote->automated_backups = 0;
remote->period = 0;
remote->curl.connection_timeout = 0;
remote->curl.transfer_timeout = 0;
Expand Down Expand Up @@ -171,15 +174,15 @@ check_remote_backup(

unsigned
check_remote(
VRT_CTX, remote_t *remote, unsigned force,
VRT_CTX, remote_t *remote, unsigned force_load, unsigned force_backup,
unsigned (*callback)(VRT_CTX, void *, char *, unsigned), void *ptr)
{
unsigned result = 0;

time_t now = time(NULL);
unsigned winner = 0;

if (!force &&
if (!force_load &&
!remote->state.reloading &&
((remote->period > 0) && (now - remote->state.tst > remote->period))) {
AZ(pthread_mutex_lock(&remote->state.mutex));
Expand All @@ -190,7 +193,7 @@ check_remote(
AZ(pthread_mutex_unlock(&remote->state.mutex));
}

if (force || winner) {
if (force_load || winner) {
char *contents = (*remote->read)(ctx, remote);

if (contents != NULL && strlen(contents) > 0) {
Expand All @@ -206,39 +209,45 @@ check_remote(
AZ(pthread_mutex_unlock(&remote->state.mutex));

if (remote->backup != NULL) {
FILE *backup = fopen(remote->backup, "wb");
if (backup != NULL) {
int rc = fputs(contents, backup);
if (rc < 0) {
// Not possible to use GNU strerror_r() due to Linux Alpine
// issue. See:
// - https://stackoverflow.com/questions/41953104/strerror-r-is-incorrectly-declared-on-alpine-linux
char buffer[256];
if (remote->automated_backups || force_backup) {
FILE *backup = fopen(remote->backup, "wb");
if (backup != NULL) {
int rc = fputs(contents, backup);
if (rc < 0) {
// Not possible to use GNU strerror_r() due to Linux Alpine
// issue. See:
// - https://stackoverflow.com/questions/41953104/strerror-r-is-incorrectly-declared-on-alpine-linux
char buffer[256];
#ifdef STRERROR_R_CHAR_P
LOG(ctx, LOG_ERR,
"Failed to write backup file (location=%s, backup=%s, error=%s)",
remote->location.raw, remote->backup, strerror_r(rc, buffer, sizeof(buffer)));
#else
rc = strerror_r(rc, buffer, sizeof(buffer));
if (rc == 0) {
LOG(ctx, LOG_ERR,
"Failed to write backup file (location=%s, backup=%s, error=%s)",
remote->location.raw, remote->backup, buffer);
remote->location.raw, remote->backup, strerror_r(rc, buffer, sizeof(buffer)));
#else
rc = strerror_r(rc, buffer, sizeof(buffer));
if (rc == 0) {
LOG(ctx, LOG_ERR,
"Failed to write backup file (location=%s, backup=%s, error=%s)",
remote->location.raw, remote->backup, buffer);
} else {
LOG(ctx, LOG_ERR,
"Failed to write backup file (location=%s, backup=%s, error=%d)",
remote->location.raw, remote->backup, rc);
}
#endif
} else {
LOG(ctx, LOG_ERR,
"Failed to write backup file (location=%s, backup=%s, error=%d)",
remote->location.raw, remote->backup, rc);
LOG(ctx, LOG_INFO,
"Successfully write to backup file (location=%s, backup=%s)",
remote->location.raw, remote->backup);
}
#endif
fclose(backup);
} else {
LOG(ctx, LOG_INFO,
"Successfully write to backup file (location=%s, backup=%s)",
LOG(ctx, LOG_ERR,
"Failed to open backup file (location=%s, backup=%s)",
remote->location.raw, remote->backup);
}
fclose(backup);
} else {
LOG(ctx, LOG_ERR,
"Failed to open backup file (location=%s, backup=%s)",
LOG(ctx, LOG_INFO,
"Automated backups are disabled (location=%s, backup=%s)",
remote->location.raw, remote->backup);
}
}
Expand Down
6 changes: 4 additions & 2 deletions src/remote.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ typedef struct remote {
const char *parsed;
} location;
const char *backup;
unsigned automated_backups;
unsigned period;
struct {
unsigned connection_timeout;
Expand All @@ -32,14 +33,15 @@ typedef struct remote {
} remote_t;

remote_t *new_remote(
const char *location, const char *backup, unsigned period, unsigned curl_connection_timeout,
const char *location, const char *backup, unsigned automated_backups,
unsigned period, unsigned curl_connection_timeout,
unsigned curl_transfer_timeout, unsigned curl_ssl_verify_peer,
unsigned curl_ssl_verify_host, const char *curl_ssl_cafile,
const char *curl_ssl_capath, const char *curl_proxy);
void free_remote(remote_t *remote);

unsigned check_remote(
VRT_CTX, remote_t *remote, unsigned force,
VRT_CTX, remote_t *remote, unsigned force_load, unsigned force_backup,
unsigned (*callback)(VRT_CTX, void *, char *, unsigned), void *ptr);

void inspect_remote(VRT_CTX, remote_t *remote);
Expand Down
66 changes: 65 additions & 1 deletion src/tests/file.backup.vtc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ varnish v_no_backup -vcl {
new file = cfg.file(
"http://${v_origin_no_backup_addr}:${v_origin_no_backup_port}/test.ini",
backup="${tmp}/backup.ini",
automated_backups=true,
period=0,
curl_connection_timeout=2000,
curl_transfer_timeout=2000,
Expand Down Expand Up @@ -101,6 +102,7 @@ varnish v_broken_backup -vcl {
new file = cfg.file(
"http://${v_origin_broken_backup_addr}:${v_origin_broken_backup_port}/test.ini",
backup="${tmp}/backup.ini",
automated_backups=true,
period=0,
curl_connection_timeout=2000,
curl_transfer_timeout=2000,
Expand Down Expand Up @@ -161,6 +163,7 @@ varnish v_valid_backup -vcl {
new file = cfg.file(
"http://${v_origin_valid_backup_addr}:${v_origin_valid_backup_port}/test.ini",
backup="${tmp}/backup.ini",
automated_backups=true,
period=0,
curl_connection_timeout=2000,
curl_transfer_timeout=2000,
Expand Down Expand Up @@ -197,6 +200,66 @@ varnish v_valid_backup -expect client_req == 1

varnish v_valid_backup -expect MGT.child_panic == 0

################################################################################
## Remote valid, backup doesn't exist and automated backups disabled
################################################################################

shell {
rm -f "${tmp}/backup.ini"
}

server s_origin_no_automated_backups {
rxreq
expect req.url == "/test.ini"
txresp -body "field: foo"
} -repeat 3 -start

varnish v_origin_no_automated_backups -vcl {
backend default {
.host = "${s_origin_no_automated_backups_addr}";
.port = "${s_origin_no_automated_backups_port}";
}

sub vcl_recv {
return (pass);
}
} -start

varnish v_no_automated_backups -vcl {
import ${vmod_cfg};

backend default {
.host = "${s_origin1_addr}";
.port = "${s_origin1_port}";
}

sub vcl_init {
new file = cfg.file(
"http://${v_origin_no_automated_backups_addr}:${v_origin_no_automated_backups_port}/test.ini",
backup="${tmp}/backup.ini",
automated_backups=false,
period=0,
curl_connection_timeout=2000,
curl_transfer_timeout=2000,
curl_ssl_verify_peer=false,
curl_ssl_verify_host=false,
curl_ssl_cafile="",
curl_ssl_capath="",
curl_proxy="",
format=ini,
name_delimiter=":",
value_delimiter=";");

file.reload(force_backup=true);
}
} -start

logexpect l_no_automated_backups -v v_no_automated_backups -d 1 -g raw {
expect * * VCL_Log {^\[CFG\].* Remote successfully parsed .*\(.*, is_backup=0, .*\)$}
expect * * VCL_Log {^\[CFG\].* Automated backups are disabled .*$}
expect * * VCL_Log {^\[CFG\].* Successfully write to backup file .*$}
} -start -wait

################################################################################
## Basic functionalities
################################################################################
Expand Down Expand Up @@ -246,6 +309,7 @@ varnish v_basics -vcl {
new file = cfg.file(
"http://${v_origin_basics_addr}:${v_origin_basics_port}/test.ini",
backup="${tmp}/backup.ini",
automated_backups=true,
period=0,
curl_connection_timeout=2000,
curl_transfer_timeout=2000,
Expand All @@ -261,7 +325,7 @@ varnish v_basics -vcl {

sub vcl_deliver {
if (req.http.reload == "1") {
set resp.http.reload = file.reload();
set resp.http.reload = file.reload(force_backup=false);
}
set resp.http.result = file.get("field", "-");
set resp.http.dump = file.dump();
Expand Down
27 changes: 24 additions & 3 deletions src/vmod_cfg.vcc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ Description
$Object file(
STRING location,
STRING backup="",
BOOL automated_backups=1,
INT period=60,
BOOL ignore_load_failures=1,
INT curl_connection_timeout=0,
Expand All @@ -81,6 +82,12 @@ Arguments
backup: when this option is used, you have to specify where to save the
backup file.

automated_backups: if enabled and a backup file has been provided, that
file is updated on every successful load of ``location`` (i.e. during
``vcl_init``, during periodical updates when ``period`` > 0, etc.).
Otherwise, the file is only updated when explicitly requested to do so (e.g.
calls to ``.reload()`` using the ``force_backup`` flag).

period: how frequently (seconds) contents of the file are reloaded (0 means
disabling periodical reloads).

Expand Down Expand Up @@ -119,8 +126,12 @@ Description
on every VCL WARM event received by the VMOD and every ``period`` seconds
(if ``period`` > 0).

$Method BOOL .reload()
$Method BOOL .reload(BOOL force_backup=1)

Arguments
force_backup: if enabled and a backup file has been provided, that file
will be updated upon a successful reload, overriding the default behavior
for automated backups.
Description
Reloads contents of the file. A ``False`` value is returned on failure.

Expand Down Expand Up @@ -167,6 +178,7 @@ Description
$Object rules(
STRING location,
STRING backup="",
BOOL automated_backups=1,
INT period=60,
BOOL ignore_load_failures=1,
INT curl_connection_timeout=0,
Expand All @@ -182,8 +194,12 @@ Description

See ``cfg.file()`` for details.

$Method BOOL .reload()
$Method BOOL .reload(BOOL force_backup=1)

Arguments
force_backup: if enabled and a backup file has been provided, that file
will be updated upon a successful reload, overriding the default behavior
for automated backups.
Description
Reloads contents of the file. A ``False`` value is returned on failure.

Expand All @@ -206,6 +222,7 @@ Description
$Object script(
STRING location="",
STRING backup="",
BOOL automated_backups=1,
INT period=60,
BOOL ignore_load_failures=1,
ENUM { lua, javascript } type="lua",
Expand Down Expand Up @@ -268,8 +285,12 @@ Description

See ``cfg.file()`` for details about undocumented arguments.

$Method BOOL .reload()
$Method BOOL .reload(BOOL force_backup=1)

Arguments
force_backup: if enabled and a backup file has been provided, that file
will be updated upon a successful reload, overriding the default behavior
for automated backups.
Description
Reloads contents of the file. A ``False`` value is returned on failure.

Expand Down
Loading

0 comments on commit 368cace

Please sign in to comment.