diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a2275866f..d18a234ba5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: build-debian-old: runs-on: ubuntu-latest - container: debian:buster + container: debian:bullseye steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: make diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 079f91b9a7..1e60649b28 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -9,6 +9,35 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Valkey 8.1.4 - Released Fri 09 October 2025 +================================================================================ + +Upgrade urgency SECURITY: This release includes security fixes we recommend you +apply as soon as possible. + +Security fixes +============== +* (CVE-2025-49844) A Lua script may lead to remote code execution +* (CVE-2025-46817) A Lua script may lead to integer overflow and potential RCE +* (CVE-2025-46818) A Lua script can be executed in the context of another user +* (CVE-2025-46819) LUA out-of-bound read + +Bug fixes +========= +* Fix accounting for dual channel RDB bytes in replication stats (#2614) +* Fix EVAL to report unknown error when empty error table is provided (#2229) +* Fix use-after-free when active expiration triggers hashtable to shrink (#2257) +* Fix MEMORY USAGE to account for embedded keys (#2290) +* Fix memory leak when shrinking a hashtable without entries (#2288) +* Prevent potential assertion in active defrag handling large allocations (#2353) +* Prevent bad memory access when NOTOUCH client gets unblocked (#2347) +* Converge divergent shard-id persisted in nodes.conf to primary's shard id (#2174) +* Fix client tracking memory overhead calculation (#2360) +* Fix RDB load per slot memory pre-allocation when loading from RDB snapshot (#2466) +* Don't use AVX2 instructions if the CPU doesn't support it (#2571) +* Fix bug where active defrag may be unable to defrag sparsely filled pages (#2656) + ================================================================================ Valkey 8.1.3 - Released Sun 07 July 2025 ================================================================================ diff --git a/deps/lua/src/lbaselib.c b/deps/lua/src/lbaselib.c index 2ab550bd48..26172d15b4 100644 --- a/deps/lua/src/lbaselib.c +++ b/deps/lua/src/lbaselib.c @@ -340,13 +340,14 @@ static int luaB_assert (lua_State *L) { static int luaB_unpack (lua_State *L) { - int i, e, n; + int i, e; + unsigned int n; luaL_checktype(L, 1, LUA_TTABLE); i = luaL_optint(L, 2, 1); e = luaL_opt(L, luaL_checkint, 3, luaL_getn(L, 1)); if (i > e) return 0; /* empty range */ - n = e - i + 1; /* number of elements */ - if (n <= 0 || !lua_checkstack(L, n)) /* n <= 0 means arith. overflow */ + n = (unsigned int)e - (unsigned int)i; /* number of elements minus 1 */ + if (n >= INT_MAX || !lua_checkstack(L, ++n)) return luaL_error(L, "too many results to unpack"); lua_rawgeti(L, 1, i); /* push arg[i] (avoiding overflow problems) */ while (i++ < e) /* push arg[i + 1...e] */ diff --git a/deps/lua/src/llex.c b/deps/lua/src/llex.c index 88c6790c07..3712edec80 100644 --- a/deps/lua/src/llex.c +++ b/deps/lua/src/llex.c @@ -138,6 +138,7 @@ static void inclinenumber (LexState *ls) { void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) { + ls->t.token = 0; ls->decpoint = '.'; ls->L = L; ls->lookahead.token = TK_EOS; /* no look-ahead token */ @@ -207,8 +208,13 @@ static void read_numeral (LexState *ls, SemInfo *seminfo) { } -static int skip_sep (LexState *ls) { - int count = 0; +/* +** reads a sequence '[=*[' or ']=*]', leaving the last bracket. +** If a sequence is well-formed, return its number of '='s + 2; otherwise, +** return 1 if there is no '='s or 0 otherwise (an unfinished '[==...'). +*/ +static size_t skip_sep (LexState *ls) { + size_t count = 0; int s = ls->current; lua_assert(s == '[' || s == ']'); save_and_next(ls); @@ -216,11 +222,13 @@ static int skip_sep (LexState *ls) { save_and_next(ls); count++; } - return (ls->current == s) ? count : (-count) - 1; + return (ls->current == s) ? count + 2 + : (count == 0) ? 1 + : 0; } -static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { +static void read_long_string (LexState *ls, SemInfo *seminfo, size_t sep) { int cont = 0; (void)(cont); /* avoid warnings when `cont' is not used */ save_and_next(ls); /* skip 2nd `[' */ @@ -270,8 +278,8 @@ static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { } } endloop: if (seminfo) - seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), - luaZ_bufflen(ls->buff) - 2*(2 + sep)); + seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + sep, + luaZ_bufflen(ls->buff) - 2 * sep); } @@ -346,9 +354,9 @@ static int llex (LexState *ls, SemInfo *seminfo) { /* else is a comment */ next(ls); if (ls->current == '[') { - int sep = skip_sep(ls); + size_t sep = skip_sep(ls); luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */ - if (sep >= 0) { + if (sep >= 2) { read_long_string(ls, NULL, sep); /* long comment */ luaZ_resetbuffer(ls->buff); continue; @@ -360,13 +368,14 @@ static int llex (LexState *ls, SemInfo *seminfo) { continue; } case '[': { - int sep = skip_sep(ls); - if (sep >= 0) { + size_t sep = skip_sep(ls); + if (sep >= 2) { read_long_string(ls, seminfo, sep); return TK_STRING; } - else if (sep == -1) return '['; - else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING); + else if (sep == 0) /* '[=...' missing second bracket */ + luaX_lexerror(ls, "invalid long string delimiter", TK_STRING); + return '['; } case '=': { next(ls); diff --git a/deps/lua/src/lparser.c b/deps/lua/src/lparser.c index dda7488dca..ee7d90c90d 100644 --- a/deps/lua/src/lparser.c +++ b/deps/lua/src/lparser.c @@ -384,13 +384,17 @@ Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, const char *name) { struct LexState lexstate; struct FuncState funcstate; lexstate.buff = buff; - luaX_setinput(L, &lexstate, z, luaS_new(L, name)); + TString *tname = luaS_new(L, name); + setsvalue2s(L, L->top, tname); + incr_top(L); + luaX_setinput(L, &lexstate, z, tname); open_func(&lexstate, &funcstate); funcstate.f->is_vararg = VARARG_ISVARARG; /* main func. is always vararg */ luaX_next(&lexstate); /* read first token */ chunk(&lexstate); check(&lexstate, TK_EOS); close_func(&lexstate); + --L->top; lua_assert(funcstate.prev == NULL); lua_assert(funcstate.f->nups == 0); lua_assert(lexstate.fs == NULL); diff --git a/deps/lua/src/ltable.c b/deps/lua/src/ltable.c index f75fe19fe3..55575a8ace 100644 --- a/deps/lua/src/ltable.c +++ b/deps/lua/src/ltable.c @@ -434,8 +434,7 @@ static TValue *newkey (lua_State *L, Table *t, const TValue *key) { ** search function for integers */ const TValue *luaH_getnum (Table *t, int key) { - /* (1 <= key && key <= t->sizearray) */ - if (cast(unsigned int, key-1) < cast(unsigned int, t->sizearray)) + if (1 <= key && key <= t->sizearray) return &t->array[key-1]; else { lua_Number nk = cast_num(key); diff --git a/src/allocator_defrag.c b/src/allocator_defrag.c index b9dedb3b07..2c05601832 100644 --- a/src/allocator_defrag.c +++ b/src/allocator_defrag.c @@ -57,7 +57,7 @@ #define SLAB_LEN(out, i) out[(i) * BATCH_QUERY_ARGS_OUT + 2] #define SLAB_NUM_REGS(out, i) out[(i) * BATCH_QUERY_ARGS_OUT + 1] -#define UTILIZATION_THRESHOLD_FACTOR_MILI (125) // 12.5% additional utilization +#define UTILIZATION_THRESHOLD_FACTOR_MILLI (125) // 12.5% additional utilization /* * Represents a precomputed key for querying jemalloc statistics. @@ -349,16 +349,26 @@ unsigned long allocatorDefragGetFragSmallbins(void) { * defragmentation is not necessary as moving regions is guaranteed not to change the fragmentation ratio. * 2. If the number of non-full slabs (bin_usage->curr_nonfull_slabs) is less than 2, defragmentation is not performed * because there is no other slab to move regions to. - * 3. If slab utilization < 'avg utilization'*1.125 [code 1.125 == (1000+UTILIZATION_THRESHOLD_FACTOR_MILI)/1000] + * 3. Defrag if the slab is less than 1/8 full to ensure small slabs get defragmented even when average utilization is low. + * This also handles the case when there are items that aren't defragmented skewing the average utilization. The 1/8 + * threshold (12.5%) was chosen to align with existing utilization threshold factor. + * 4. If slab utilization < 'avg utilization'*1.125 [code 1.125 == (1000+UTILIZATION_THRESHOLD_FACTOR_MILLI)/1000] * than we should defrag. This is aligned with previous je_defrag_hint implementation. */ static inline int makeDefragDecision(jeBinInfo *bin_info, jemallocBinUsageData *bin_usage, unsigned long nalloced) { unsigned long curr_full_slabs = bin_usage->curr_slabs - bin_usage->curr_nonfull_slabs; size_t allocated_nonfull = bin_usage->curr_regs - curr_full_slabs * bin_info->nregs; - if (bin_info->nregs == nalloced || bin_usage->curr_nonfull_slabs < 2 || - 1000 * nalloced * bin_usage->curr_nonfull_slabs > (1000 + UTILIZATION_THRESHOLD_FACTOR_MILI) * allocated_nonfull) { - return 0; - } + + /* Don't defrag if the slab is full or if there's only 1 nonfull slab */ + if (bin_info->nregs == nalloced || bin_usage->curr_nonfull_slabs < 2) return 0; + + /* Defrag if the slab is less than 1/8 full */ + if (1000 * nalloced < bin_info->nregs * UTILIZATION_THRESHOLD_FACTOR_MILLI) return 1; + + /* Don't defrag if the slab usage is greater than the average usage (+ 12.5%) */ + if (1000 * nalloced * bin_usage->curr_nonfull_slabs > (1000 + UTILIZATION_THRESHOLD_FACTOR_MILLI) * allocated_nonfull) return 0; + + /* Otherwise, defrag! */ return 1; } @@ -385,7 +395,7 @@ int allocatorShouldDefrag(void *ptr) { assert(SLAB_NUM_REGS(out, 0) > 0); assert(SLAB_LEN(out, 0) > 0); assert(SLAB_NFREE(out, 0) != (size_t)-1); - unsigned region_size = SLAB_LEN(out, 0) / SLAB_NUM_REGS(out, 0); + size_t region_size = SLAB_LEN(out, 0) / SLAB_NUM_REGS(out, 0); /* check that the allocation size is in range of small bins */ if (region_size > je_cb.bin_info[je_cb.nbins - 1].reg_size) { return 0; diff --git a/src/bio.c b/src/bio.c index e55c729f74..ab07581ba9 100644 --- a/src/bio.c +++ b/src/bio.c @@ -142,8 +142,9 @@ void bioInit(void) { * responsible for. */ for (j = 0; j < BIO_WORKER_NUM; j++) { void *arg = (void *)(unsigned long)j; - if (pthread_create(&thread, &attr, bioProcessBackgroundJobs, arg) != 0) { - serverLog(LL_WARNING, "Fatal: Can't initialize Background Jobs. Error message: %s", strerror(errno)); + int err = pthread_create(&thread, &attr, bioProcessBackgroundJobs, arg); + if (err) { + serverLog(LL_WARNING, "Fatal: Can't initialize Background Jobs. Error message: %s", strerror(err)); exit(1); } bio_threads[j] = thread; @@ -222,8 +223,9 @@ void *bioProcessBackgroundJobs(void *arg) { * receive the watchdog signal. */ sigemptyset(&sigset); sigaddset(&sigset, SIGALRM); - if (pthread_sigmask(SIG_BLOCK, &sigset, NULL)) - serverLog(LL_WARNING, "Warning: can't mask SIGALRM in bio.c thread: %s", strerror(errno)); + int err = pthread_sigmask(SIG_BLOCK, &sigset, NULL); + if (err) + serverLog(LL_WARNING, "Warning: can't mask SIGALRM in bio.c thread: %s", strerror(err)); while (1) { listNode *ln; diff --git a/src/bitops.c b/src/bitops.c index af34806ccf..ff4ba2bb46 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -194,7 +194,7 @@ long long serverPopcount(void *s, long count) { #ifdef HAVE_AVX2 /* If length of s >= 256 bits and the CPU supports AVX2, * we prefer to use the SIMD version */ - if (count >= 32) { + if (count >= 32 && __builtin_cpu_supports("avx2")) { return popcountAVX2(s, count); } #endif diff --git a/src/childinfo.c b/src/childinfo.c index f9a90a23cc..d9946e1835 100644 --- a/src/childinfo.c +++ b/src/childinfo.c @@ -36,6 +36,7 @@ typedef struct { size_t cow; monotime cow_updated; double progress; + size_t repl_output_bytes; childInfoType information_type; /* Type of information */ } child_info_data; @@ -64,7 +65,7 @@ void closeChildInfoPipe(void) { } /* Send save data to parent. */ -void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, char *pname) { +void sendChildInfoGeneric(childInfoType info_type, size_t keys, size_t repl_output_bytes, double progress, char *pname) { if (server.child_info_pipe[1] == -1) return; static monotime cow_updated = 0; @@ -101,6 +102,7 @@ void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, data.information_type = info_type; data.keys = keys; + data.repl_output_bytes = repl_output_bytes; data.cow = cow; data.cow_updated = cow_updated; data.progress = progress; @@ -115,7 +117,7 @@ void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, } /* Update Child info. */ -void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_updated, size_t keys, double progress) { +void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_updated, size_t keys, size_t repl_output_bytes, double progress) { if (cow > server.stat_current_cow_peak) server.stat_current_cow_peak = cow; if (information_type == CHILD_INFO_TYPE_CURRENT_INFO) { @@ -129,6 +131,8 @@ void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_up server.stat_rdb_cow_bytes = server.stat_current_cow_peak; } else if (information_type == CHILD_INFO_TYPE_MODULE_COW_SIZE) { server.stat_module_cow_bytes = server.stat_current_cow_peak; + } else if (information_type == CHILD_INFO_TYPE_REPL_OUTPUT_BYTES) { + server.stat_net_repl_output_bytes += (long long)repl_output_bytes; } } @@ -136,7 +140,7 @@ void updateChildInfo(childInfoType information_type, size_t cow, monotime cow_up * if complete data read into the buffer, * data is stored into *buffer, and returns 1. * otherwise, the partial data is left in the buffer, waiting for the next read, and returns 0. */ -int readChildInfo(childInfoType *information_type, size_t *cow, monotime *cow_updated, size_t *keys, double *progress) { +int readChildInfo(childInfoType *information_type, size_t *cow, monotime *cow_updated, size_t *keys, size_t *repl_output_bytes, double *progress) { /* We are using here a static buffer in combination with the server.child_info_nread to handle short reads */ static child_info_data buffer; ssize_t wlen = sizeof(buffer); @@ -156,6 +160,7 @@ int readChildInfo(childInfoType *information_type, size_t *cow, monotime *cow_up *cow = buffer.cow; *cow_updated = buffer.cow_updated; *keys = buffer.keys; + *repl_output_bytes = buffer.repl_output_bytes; *progress = buffer.progress; return 1; } else { @@ -170,11 +175,12 @@ void receiveChildInfo(void) { size_t cow; monotime cow_updated; size_t keys; + size_t repl_output_bytes; double progress; childInfoType information_type; /* Drain the pipe and update child info so that we get the final message. */ - while (readChildInfo(&information_type, &cow, &cow_updated, &keys, &progress)) { - updateChildInfo(information_type, cow, cow_updated, keys, progress); + while (readChildInfo(&information_type, &cow, &cow_updated, &keys, &repl_output_bytes, &progress)) { + updateChildInfo(information_type, cow, cow_updated, keys, repl_output_bytes, progress); } } diff --git a/src/cluster_legacy.c b/src/cluster_legacy.c index 89040f532b..969ac04192 100644 --- a/src/cluster_legacy.c +++ b/src/cluster_legacy.c @@ -316,10 +316,16 @@ int auxShardIdSetter(clusterNode *n, void *value, size_t length) { } memcpy(n->shard_id, value, CLUSTER_NAMELEN); /* if n already has replicas, make sure they all agree - * on the shard id */ + * on the shard id. If not, update them. */ for (int i = 0; i < n->num_replicas; i++) { if (memcmp(n->replicas[i]->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0) { - return C_ERR; + serverLog(LL_NOTICE, + "Node %.40s has a different shard id (%.40s) than its primary's shard id %.40s (%.40s). " + "Updating replica's shard id to match primary's shard id.", + n->replicas[i]->name, n->replicas[i]->shard_id, n->name, n->shard_id); + clusterRemoveNodeFromShard(n->replicas[i]); + memcpy(n->replicas[i]->shard_id, n->shard_id, CLUSTER_NAMELEN); + clusterAddNodeToShard(n->shard_id, n->replicas[i]); } } clusterAddNodeToShard(value, n); @@ -720,10 +726,16 @@ int clusterLoadConfig(char *filename) { clusterAddNodeToShard(primary->shard_id, n); } else if (clusterGetNodesInMyShard(primary) != NULL && memcmp(primary->shard_id, n->shard_id, CLUSTER_NAMELEN) != 0) { - /* If the primary has been added to a shard, make sure this - * node has the same persisted shard id as the primary. */ - sdsfreesplitres(argv, argc); - goto fmterr; + /* If the primary has been added to a shard and this replica has + * a different shard id stored in nodes.conf, update it to match + * the primary instead of aborting the startup. */ + serverLog(LL_NOTICE, + "Node %.40s has a different shard id (%.40s) than its primary %.40s (%.40s). " + "Updating replica's shard id to match primary's shard id.", + n->name, n->shard_id, primary->name, primary->shard_id); + clusterRemoveNodeFromShard(n); + memcpy(n->shard_id, primary->shard_id, CLUSTER_NAMELEN); + clusterAddNodeToShard(primary->shard_id, n); } n->replicaof = primary; clusterNodeAddReplica(primary, n); @@ -4918,6 +4930,7 @@ void clusterHandleReplicaFailover(void) { if (server.cluster->mf_end) { server.cluster->failover_auth_time = now; server.cluster->failover_auth_rank = 0; + server.cluster->failover_failed_primary_rank = 0; /* Reset auth_age since it is outdated now and we can bypass the auth_timeout * check in the next state and start the election ASAP. */ auth_age = 0; diff --git a/src/config.c b/src/config.c index 14b902347b..1566a799e4 100644 --- a/src/config.c +++ b/src/config.c @@ -33,6 +33,7 @@ #include "connection.h" #include "bio.h" #include "module.h" +#include "eval.h" #include #include @@ -2596,6 +2597,15 @@ int invalidateClusterSlotsResp(const char **err) { return 1; } +static int updateLuaEnableInsecureApi(const char **err) { + UNUSED(err); + if (server.lua_insecure_api_current != server.lua_enable_insecure_api) { + evalReset(server.lazyfree_lazy_user_flush ? 1 : 0); + } + server.lua_insecure_api_current = server.lua_enable_insecure_api; + return 1; +} + int updateRequirePass(const char **err) { UNUSED(err); /* The old "requirepass" directive just translates to setting @@ -3193,6 +3203,7 @@ standardConfig static_configs[] = { createBoolConfig("enable-debug-assert", NULL, IMMUTABLE_CONFIG | HIDDEN_CONFIG, server.enable_debug_assert, 0, NULL, NULL), createBoolConfig("cluster-slot-stats-enabled", NULL, MODIFIABLE_CONFIG, server.cluster_slot_stats_enabled, 0, NULL, NULL), createBoolConfig("hide-user-data-from-log", NULL, MODIFIABLE_CONFIG, server.hide_user_data_from_log, 1, NULL, NULL), + createBoolConfig("lua-enable-insecure-api", "lua-enable-deprecated-api", MODIFIABLE_CONFIG | HIDDEN_CONFIG | PROTECTED_CONFIG, server.lua_enable_insecure_api, 0, NULL, updateLuaEnableInsecureApi), createBoolConfig("import-mode", NULL, DEBUG_CONFIG | MODIFIABLE_CONFIG, server.import_mode, 0, NULL, NULL), /* String Configs */ diff --git a/src/db.c b/src/db.c index 036e16d079..4e48781211 100644 --- a/src/db.c +++ b/src/db.c @@ -124,8 +124,9 @@ robj *lookupKey(serverDb *db, robj *key, int flags) { /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ - if (server.current_client && server.current_client->flag.no_touch && - server.executing_client->cmd->proc != touchCommand) + if ((flags & LOOKUP_NOTOUCH) == 0 && + server.current_client && server.current_client->flag.no_touch && + server.executing_client && server.executing_client->cmd->proc != touchCommand) flags |= LOOKUP_NOTOUCH; if (!hasActiveChildProcess() && !(flags & LOOKUP_NOTOUCH)) { /* Shared objects can't be stored in the database. */ diff --git a/src/eval.h b/src/eval.h index 4f30b3d372..f9c217ef0d 100644 --- a/src/eval.h +++ b/src/eval.h @@ -2,7 +2,7 @@ #define _EVAL_H_ void evalInit(void); - +void evalReset(int async); void *evalActiveDefragScript(void *ptr); #endif /* _EVAL_H_ */ diff --git a/src/hashtable.c b/src/hashtable.c index ee7e734dcc..8d7da9c117 100644 --- a/src/hashtable.c +++ b/src/hashtable.c @@ -649,7 +649,7 @@ static int resize(hashtable *ht, size_t min_capacity, int *malloc_failed) { if (ht->type->rehashingStarted) ht->type->rehashingStarted(ht); /* If the old table was empty, the rehashing is completed immediately. */ - if (ht->tables[0] == NULL || ht->used[0] == 0) { + if (ht->tables[0] == NULL || (ht->used[0] == 0 && ht->child_buckets[0] == 0)) { rehashingCompleted(ht); } else if (ht->type->instant_rehashing) { while (hashtableIsRehashing(ht)) { @@ -1218,7 +1218,7 @@ int hashtableExpandIfNeeded(hashtable *ht) { * resize policy to ALLOW, you may want to call hashtableShrinkIfNeeded. */ int hashtableShrinkIfNeeded(hashtable *ht) { /* Don't shrink if rehashing is already in progress. */ - if (hashtableIsRehashing(ht) || resize_policy == HASHTABLE_RESIZE_FORBID) { + if (hashtableIsRehashing(ht) || resize_policy == HASHTABLE_RESIZE_FORBID || ht->pause_auto_shrink) { return 0; } size_t current_capacity = numBuckets(ht->bucket_exp[0]) * ENTRIES_PER_BUCKET; @@ -1710,6 +1710,7 @@ size_t hashtableScanDefrag(hashtable *ht, size_t cursor, hashtableScanFunction f /* Prevent entries from being moved around during the scan call, as a * side-effect of the scan callback. */ hashtablePauseRehashing(ht); + hashtablePauseAutoShrink(ht); /* Flags. */ int emit_ref = (flags & HASHTABLE_SCAN_EMIT_REF); @@ -1717,7 +1718,9 @@ size_t hashtableScanDefrag(hashtable *ht, size_t cursor, hashtableScanFunction f if (!hashtableIsRehashing(ht)) { /* Emit entries at the cursor index. */ size_t mask = expToMask(ht->bucket_exp[0]); - bucket *b = &ht->tables[0][cursor & mask]; + size_t idx = cursor & mask; + size_t used_before = ht->used[0]; + bucket *b = &ht->tables[0][idx]; do { if (b->presence != 0) { int pos; @@ -1735,6 +1738,11 @@ size_t hashtableScanDefrag(hashtable *ht, size_t cursor, hashtableScanFunction f b = next; } while (b != NULL); + /* If any entries were deleted, fill the holes. */ + if (ht->used[0] < used_before) { + compactBucketChain(ht, idx, 0); + } + /* Advance cursor. */ cursor = nextCursor(cursor, mask); } else { @@ -1814,6 +1822,7 @@ size_t hashtableScanDefrag(hashtable *ht, size_t cursor, hashtableScanFunction f } while (cursor & (mask_small ^ mask_large)); } hashtableResumeRehashing(ht); + hashtableResumeAutoShrink(ht); return cursor; } diff --git a/src/io_threads.c b/src/io_threads.c index 6c5a878ca9..f713ba447f 100644 --- a/src/io_threads.c +++ b/src/io_threads.c @@ -265,8 +265,9 @@ static void createIOThread(int id) { pthread_mutex_init(&io_threads_mutex[id], NULL); IOJobQueue_init(&io_jobs[id], IO_JOB_QUEUE_SIZE); pthread_mutex_lock(&io_threads_mutex[id]); /* Thread will be stopped. */ - if (pthread_create(&tid, NULL, IOThreadMain, (void *)(long)id) != 0) { - serverLog(LL_WARNING, "Fatal: Can't initialize IO thread, pthread_create failed with: %s", strerror(errno)); + int err = pthread_create(&tid, NULL, IOThreadMain, (void *)(long)id); + if (err) { + serverLog(LL_WARNING, "Fatal: Can't initialize IO thread, pthread_create failed with: %s", strerror(err)); exit(1); } io_threads[id] = tid; diff --git a/src/kvstore.c b/src/kvstore.c index 486e434e1a..066f7716b6 100644 --- a/src/kvstore.c +++ b/src/kvstore.c @@ -429,12 +429,10 @@ int kvstoreExpand(kvstore *kvs, uint64_t newsize, int try_expand, kvstoreExpandS if (newsize == 0) return 1; for (int i = 0; i < kvs->num_hashtables; i++) { if (skip_cb && skip_cb(i)) continue; - /* If the hash table doesn't exist, create it. */ - hashtable *ht = createHashtableIfNeeded(kvs, i); if (try_expand) { - if (!hashtableTryExpand(ht, newsize)) return 0; + if (!kvstoreHashtableTryExpand(kvs, i, newsize)) return 0; } else { - hashtableExpand(ht, newsize); + kvstoreHashtableExpand(kvs, i, newsize); } } @@ -684,6 +682,12 @@ unsigned long kvstoreHashtableSize(kvstore *kvs, int didx) { return hashtableSize(ht); } +unsigned long kvstoreHashtableBuckets(kvstore *kvs, int didx) { + hashtable *ht = kvstoreGetHashtable(kvs, didx); + if (!ht) return 0; + return hashtableBuckets(ht); +} + kvstoreHashtableIterator *kvstoreGetHashtableIterator(kvstore *kvs, int didx, uint8_t flags) { kvstoreHashtableIterator *kvs_di = zmalloc(sizeof(*kvs_di)); kvs_di->kvs = kvs; @@ -731,11 +735,17 @@ unsigned int kvstoreHashtableSampleEntries(kvstore *kvs, int didx, void **dst, u } int kvstoreHashtableExpand(kvstore *kvs, int didx, unsigned long size) { - hashtable *ht = kvstoreGetHashtable(kvs, didx); - if (!ht) return 0; + if (size == 0) return 0; + hashtable *ht = createHashtableIfNeeded(kvs, didx); return hashtableExpand(ht, size); } +int kvstoreHashtableTryExpand(kvstore *kvs, int didx, unsigned long size) { + if (size == 0) return 0; + hashtable *ht = createHashtableIfNeeded(kvs, didx); + return hashtableTryExpand(ht, size); +} + unsigned long kvstoreHashtableScanDefrag(kvstore *kvs, int didx, unsigned long v, diff --git a/src/kvstore.h b/src/kvstore.h index d5db1a89aa..857868b805 100644 --- a/src/kvstore.h +++ b/src/kvstore.h @@ -57,6 +57,7 @@ unsigned long kvstoreHashtableRehashingCount(kvstore *kvs); /* Specific hashtable access by hashtable-index */ unsigned long kvstoreHashtableSize(kvstore *kvs, int didx); +unsigned long kvstoreHashtableBuckets(kvstore *kvs, int didx); kvstoreHashtableIterator *kvstoreGetHashtableIterator(kvstore *kvs, int didx, uint8_t flags); void kvstoreReleaseHashtableIterator(kvstoreHashtableIterator *kvs_id); int kvstoreHashtableIteratorNext(kvstoreHashtableIterator *kvs_di, void **next); @@ -64,6 +65,7 @@ int kvstoreHashtableRandomEntry(kvstore *kvs, int didx, void **found); int kvstoreHashtableFairRandomEntry(kvstore *kvs, int didx, void **found); unsigned int kvstoreHashtableSampleEntries(kvstore *kvs, int didx, void **dst, unsigned int count); int kvstoreHashtableExpand(kvstore *kvs, int didx, unsigned long size); +int kvstoreHashtableTryExpand(kvstore *kvs, int didx, unsigned long size); unsigned long kvstoreHashtableScanDefrag(kvstore *kvs, int didx, unsigned long v, diff --git a/src/lua/engine_lua.c b/src/lua/engine_lua.c index 3d1b984138..d628f0413d 100644 --- a/src/lua/engine_lua.c +++ b/src/lua/engine_lua.c @@ -104,6 +104,8 @@ static void luaStateLockGlobalTable(lua_State *lua) { /* Recursively lock all tables that can be reached from the global table */ luaSetTableProtectionRecursively(lua); lua_pop(lua, 1); + /* Set metatables of basic types (string, number, nil etc.) readonly. */ + luaSetTableProtectionForBasicTypes(lua); } diff --git a/src/lua/function_lua.c b/src/lua/function_lua.c index 1449c812e9..d4fb00a8ef 100644 --- a/src/lua/function_lua.c +++ b/src/lua/function_lua.c @@ -435,6 +435,8 @@ void luaFunctionInitializeLuaState(lua_State *lua) { lua_setmetatable(lua, -2); lua_enablereadonlytable(lua, -1, 1); /* protect the new global table */ lua_replace(lua, LUA_GLOBALSINDEX); /* set new global table as the new globals */ + /* Set metatables of basic types (string, number, nil etc.) readonly. */ + luaSetTableProtectionForBasicTypes(lua); } void luaFunctionFreeFunction(lua_State *lua, void *function) { diff --git a/src/lua/script_lua.c b/src/lua/script_lua.c index 1791c1f66e..fa3a96fda3 100644 --- a/src/lua/script_lua.c +++ b/src/lua/script_lua.c @@ -70,7 +70,6 @@ static char *server_api_allow_list[] = { static char *lua_builtins_allow_list[] = { "xpcall", "tostring", - "getfenv", "setmetatable", "next", "assert", @@ -91,15 +90,16 @@ static char *lua_builtins_allow_list[] = { "loadstring", "ipairs", "_VERSION", - "setfenv", "load", "error", NULL, }; -/* Lua builtins which are not documented on the Lua documentation */ -static char *lua_builtins_not_documented_allow_list[] = { +/* Lua builtins which are deprecated for sandboxing concerns */ +static char *lua_builtins_deprecated[] = { "newproxy", + "setfenv", + "getfenv", NULL, }; @@ -121,7 +121,6 @@ static char **allow_lists[] = { libraries_allow_list, server_api_allow_list, lua_builtins_allow_list, - lua_builtins_not_documented_allow_list, lua_builtins_removed_after_initialization_allow_list, NULL, }; @@ -1325,7 +1324,21 @@ static int luaNewIndexAllowList(lua_State *lua) { break; } } - if (!*allow_l) { + int allowed = (*allow_l != NULL); + /* If not explicitly allowed, check if it's a deprecated function. If so, + * allow it only if 'lua_enable_insecure_api' config is enabled. */ + int deprecated = 0; + if (!allowed) { + char **c = lua_builtins_deprecated; + for (; *c; ++c) { + if (strcmp(*c, variable_name) == 0) { + deprecated = 1; + allowed = server.lua_enable_insecure_api ? 1 : 0; + break; + } + } + } + if (!allowed) { /* Search the value on the back list, if its there we know that it was removed * on purpose and there is no need to print a warning. */ char **c = deny_list; @@ -1334,7 +1347,7 @@ static int luaNewIndexAllowList(lua_State *lua) { break; } } - if (!*c) { + if (!*c && !deprecated) { serverLog(LL_WARNING, "A key '%s' was added to Lua globals which is not on the globals allow list nor listed on the " "deny list.", @@ -1389,6 +1402,36 @@ void luaSetTableProtectionRecursively(lua_State *lua) { } } +/* Set the readonly flag on the metatable of basic types (string, nil etc.) */ +void luaSetTableProtectionForBasicTypes(lua_State *lua) { + static const int types[] = { + LUA_TSTRING, + LUA_TNUMBER, + LUA_TBOOLEAN, + LUA_TNIL, + LUA_TFUNCTION, + LUA_TTHREAD, + LUA_TLIGHTUSERDATA}; + + for (size_t i = 0; i < sizeof(types) / sizeof(types[0]); i++) { + /* Push a dummy value of the type to get its metatable */ + switch (types[i]) { + case LUA_TSTRING: lua_pushstring(lua, ""); break; + case LUA_TNUMBER: lua_pushnumber(lua, 0); break; + case LUA_TBOOLEAN: lua_pushboolean(lua, 0); break; + case LUA_TNIL: lua_pushnil(lua); break; + case LUA_TFUNCTION: lua_pushcfunction(lua, NULL); break; + case LUA_TTHREAD: lua_newthread(lua); break; + case LUA_TLIGHTUSERDATA: lua_pushlightuserdata(lua, (void *)lua); break; + } + if (lua_getmetatable(lua, -1)) { + luaSetTableProtectionRecursively(lua); + lua_pop(lua, 1); /* pop metatable */ + } + lua_pop(lua, 1); /* pop dummy value */ + } +} + void luaRegisterVersion(lua_State *lua) { /* For legacy compatibility reasons include Redis versions. */ lua_pushstring(lua, "REDIS_VERSION_NUM"); @@ -1668,6 +1711,11 @@ void luaExtractErrorInformation(lua_State *lua, errorInfo *err_info) { err_info->ignore_err_stats_update = lua_toboolean(lua, -1); } lua_pop(lua, 1); + + if (err_info->msg == NULL) { + /* Ensure we never return a NULL msg. */ + err_info->msg = sdsnew("ERR unknown error"); + } } /* This is the core of our Lua debugger, called each time Lua is about diff --git a/src/lua/script_lua.h b/src/lua/script_lua.h index 1eb40d77af..3ecbdf44c0 100644 --- a/src/lua/script_lua.h +++ b/src/lua/script_lua.h @@ -71,6 +71,7 @@ void luaRegisterGlobalProtectionFunction(lua_State *lua); void luaSetErrorMetatable(lua_State *lua); void luaSetAllowListProtection(lua_State *lua); void luaSetTableProtectionRecursively(lua_State *lua); +void luaSetTableProtectionForBasicTypes(lua_State *lua); void luaRegisterLogFunction(lua_State *lua); void luaRegisterVersion(lua_State *lua); void luaPushErrorBuff(lua_State *lua, sds err_buff); diff --git a/src/module.c b/src/module.c index ed5cb0db6c..8dae6937d7 100644 --- a/src/module.c +++ b/src/module.c @@ -11388,7 +11388,7 @@ int VM_Fork(ValkeyModuleForkDoneHandler cb, void *user_data) { * reported in INFO. * The `progress` argument should between 0 and 1, or -1 when not available. */ void VM_SendChildHeartbeat(double progress) { - sendChildInfoGeneric(CHILD_INFO_TYPE_CURRENT_INFO, 0, progress, "Module fork"); + sendChildInfoGeneric(CHILD_INFO_TYPE_CURRENT_INFO, 0, 0, progress, "Module fork"); } /* Call from the child process when you want to terminate it. diff --git a/src/networking.c b/src/networking.c index 7e10219457..841754cbfc 100644 --- a/src/networking.c +++ b/src/networking.c @@ -859,7 +859,7 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) { * - It has enough room already allocated * - And not too large (avoid large memmove) * - And the client is not in a pending I/O state */ - if (ln->prev != NULL && (prev = listNodeValue(ln->prev)) && prev->size - prev->used > 0 && + if (ln->prev != NULL && (prev = listNodeValue(ln->prev)) && prev->used < prev->size && c->io_write_state != CLIENT_PENDING_IO) { size_t len_to_copy = prev->size - prev->used; if (len_to_copy > length) len_to_copy = length; @@ -1319,7 +1319,10 @@ inline int isDeferredReplyEnabled(client *c) { * callback. */ void initDeferredReplyBuffer(client *c) { if (moduleNotifyKeyspaceSubscribersCnt() == 0) return; - if (c->deferred_reply == NULL) c->deferred_reply = listCreate(); + if (c->deferred_reply == NULL) { + c->deferred_reply = listCreate(); + listSetFreeMethod(c->deferred_reply, freeClientReplyValue); + } if (!isDeferredReplyEnabled(c)) c->deferred_reply_bytes = 0; } @@ -2079,6 +2082,11 @@ static void writeToReplica(client *c) { size_t len = (cur_node == last_node) ? bufpos : cur_block->used; len -= start; + /* For TLS, we should not call SSL_write() with num=0 */ + if (unlikely(len == 0)) { + continue; + } + iov[iovcnt].iov_base = cur_block->buf + start; iov[iovcnt].iov_len = len; total_bytes += len; @@ -4642,7 +4650,7 @@ size_t getClientMemoryUsage(client *c, size_t *output_buffer_mem_usage) { /* Add memory overhead of the tracking prefixes, this is an underestimation so we don't need to traverse the entire * rax */ if (c->pubsub_data && c->pubsub_data->client_tracking_prefixes) - mem += c->pubsub_data->client_tracking_prefixes->numnodes * (sizeof(raxNode) * sizeof(raxNode *)); + mem += c->pubsub_data->client_tracking_prefixes->numnodes * (sizeof(raxNode) + sizeof(raxNode *)); return mem; } diff --git a/src/object.c b/src/object.c index fccd1f1e5b..9ddd412980 100644 --- a/src/object.c +++ b/src/object.c @@ -1115,37 +1115,34 @@ char *strEncoding(int encoding) { * are checked and averaged to estimate the total size. */ #define OBJ_COMPUTE_SIZE_DEF_SAMPLES 5 /* Default sample size. */ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { - size_t asize = 0, elesize = 0, samples = 0; + size_t elesize = 0, samples = 0; + size_t asize = zmalloc_size((void *)o); if (o->type == OBJ_STRING) { - if (o->encoding == OBJ_ENCODING_INT) { - asize = sizeof(*o); - } else if (o->encoding == OBJ_ENCODING_RAW) { - asize = sdsAllocSize(o->ptr) + sizeof(*o); - } else if (o->encoding == OBJ_ENCODING_EMBSTR) { - asize = zmalloc_size((void *)o); - } else { + if (o->encoding == OBJ_ENCODING_RAW) { + asize += sdsAllocSize(o->ptr); + } else if (o->encoding != OBJ_ENCODING_INT && o->encoding != OBJ_ENCODING_EMBSTR) { serverPanic("Unknown string encoding"); } } else if (o->type == OBJ_LIST) { if (o->encoding == OBJ_ENCODING_QUICKLIST) { quicklist *ql = o->ptr; quicklistNode *node = ql->head; - asize = sizeof(*o) + sizeof(quicklist); + asize += sizeof(quicklist); do { elesize += sizeof(quicklistNode) + zmalloc_size(node->entry); samples++; } while ((node = node->next) && samples < sample_size); asize += (double)elesize / samples * ql->len; } else if (o->encoding == OBJ_ENCODING_LISTPACK) { - asize = sizeof(*o) + zmalloc_size(o->ptr); + asize += zmalloc_size(o->ptr); } else { serverPanic("Unknown list encoding"); } } else if (o->type == OBJ_SET) { if (o->encoding == OBJ_ENCODING_HASHTABLE) { hashtable *ht = o->ptr; - asize = sizeof(*o) + hashtableMemUsage(ht); + asize += hashtableMemUsage(ht); hashtableIterator iter; hashtableInitIterator(&iter, ht, 0); @@ -1158,21 +1155,21 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { hashtableResetIterator(&iter); if (samples) asize += (double)elesize / samples * hashtableSize(ht); } else if (o->encoding == OBJ_ENCODING_INTSET) { - asize = sizeof(*o) + zmalloc_size(o->ptr); + asize += zmalloc_size(o->ptr); } else if (o->encoding == OBJ_ENCODING_LISTPACK) { - asize = sizeof(*o) + zmalloc_size(o->ptr); + asize += zmalloc_size(o->ptr); } else { serverPanic("Unknown set encoding"); } } else if (o->type == OBJ_ZSET) { if (o->encoding == OBJ_ENCODING_LISTPACK) { - asize = sizeof(*o) + zmalloc_size(o->ptr); + asize += zmalloc_size(o->ptr); } else if (o->encoding == OBJ_ENCODING_SKIPLIST) { hashtable *ht = ((zset *)o->ptr)->ht; zskiplist *zsl = ((zset *)o->ptr)->zsl; zskiplistNode *znode = zsl->header->level[0].forward; - asize = sizeof(*o) + sizeof(zset) + sizeof(zskiplist) + - hashtableMemUsage(ht) + zmalloc_size(zsl->header); + asize += sizeof(zset) + sizeof(zskiplist) + + hashtableMemUsage(ht) + zmalloc_size(zsl->header); while (znode != NULL && samples < sample_size) { elesize += sdsAllocSize(znode->ele); elesize += zmalloc_size(znode); @@ -1185,14 +1182,14 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { } } else if (o->type == OBJ_HASH) { if (o->encoding == OBJ_ENCODING_LISTPACK) { - asize = sizeof(*o) + zmalloc_size(o->ptr); + asize += zmalloc_size(o->ptr); } else if (o->encoding == OBJ_ENCODING_HASHTABLE) { hashtable *ht = o->ptr; hashtableIterator iter; hashtableInitIterator(&iter, ht, 0); void *next; - asize = sizeof(*o) + hashtableMemUsage(ht); + asize += hashtableMemUsage(ht); while (hashtableNext(&iter, &next) && samples < sample_size) { elesize += hashTypeEntryMemUsage(next); samples++; @@ -1204,7 +1201,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { } } else if (o->type == OBJ_STREAM) { stream *s = o->ptr; - asize = sizeof(*o) + sizeof(*s); + asize += sizeof(*s); asize += raxAllocSize(s->rax); /* Now we have to add the listpacks. The last listpack is often non @@ -1274,7 +1271,7 @@ size_t objectComputeSize(robj *key, robj *o, size_t sample_size, int dbid) { if (samples) asize += (double)elesize / samples * raxSize(s->cgroups); } } else if (o->type == OBJ_MODULE) { - asize = moduleGetMemUsage(key, o, sample_size, dbid); + asize += moduleGetMemUsage(key, o, sample_size, dbid); } else { serverPanic("Unknown object type"); } diff --git a/src/rdb.c b/src/rdb.c index 84c9eb01a7..07c6af67ab 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -3179,8 +3179,8 @@ int rdbLoadRioWithLoadingCtx(rio *rdb, int rdbflags, rdbSaveInfo *rsi, rdbLoadin if (server.cluster_enabled) { /* In cluster mode we resize individual slot specific dictionaries based on the number of keys that * slot holds. */ - kvstoreHashtableExpand(db->keys, slot_id, slot_size); - kvstoreHashtableExpand(db->expires, slot_id, expires_slot_size); + if (slot_size) kvstoreHashtableExpand(db->keys, slot_id, slot_size); + if (expires_slot_size) kvstoreHashtableExpand(db->expires, slot_id, expires_slot_size); should_expand_db = 0; } } else { @@ -3652,6 +3652,9 @@ int rdbSaveToReplicasSockets(int req, rdbSaveInfo *rsi) { if (retval == C_OK) { sendChildCowInfo(CHILD_INFO_TYPE_RDB_COW_SIZE, "RDB"); + if (dual_channel) { + sendChildInfoGeneric(CHILD_INFO_TYPE_REPL_OUTPUT_BYTES, 0, rdb.processed_bytes, -1, "RDB"); + } } if (dual_channel) { rioFreeConnset(&rdb); diff --git a/src/replication.c b/src/replication.c index d6a06bb967..7116b2b5ff 100644 --- a/src/replication.c +++ b/src/replication.c @@ -3812,7 +3812,7 @@ void syncWithPrimary(connection *conn) { if (connConnect(server.repl_rdb_transfer_s, server.primary_host, server.primary_port, server.bind_source_addr, dualChannelFullSyncWithPrimary) == C_ERR) { dualChannelServerLog(LL_WARNING, "Unable to connect to Primary: %s", - connGetLastError(server.repl_transfer_s)); + connGetLastError(server.repl_rdb_transfer_s)); connClose(server.repl_rdb_transfer_s); server.repl_rdb_transfer_s = NULL; goto error; diff --git a/src/server.c b/src/server.c index 8e30ad6987..e30553b904 100644 --- a/src/server.c +++ b/src/server.c @@ -6603,11 +6603,11 @@ int serverFork(int purpose) { } void sendChildCowInfo(childInfoType info_type, char *pname) { - sendChildInfoGeneric(info_type, 0, -1, pname); + sendChildInfoGeneric(info_type, 0, 0, -1, pname); } void sendChildInfo(childInfoType info_type, size_t keys, char *pname) { - sendChildInfoGeneric(info_type, keys, -1, pname); + sendChildInfoGeneric(info_type, keys, 0, -1, pname); } /* Dismiss big chunks of memory inside a client structure, see zmadvise_dontneed() */ diff --git a/src/server.h b/src/server.h index e1697305d0..f5ecd6340d 100644 --- a/src/server.h +++ b/src/server.h @@ -1562,7 +1562,8 @@ typedef enum childInfoType { CHILD_INFO_TYPE_CURRENT_INFO, CHILD_INFO_TYPE_AOF_COW_SIZE, CHILD_INFO_TYPE_RDB_COW_SIZE, - CHILD_INFO_TYPE_MODULE_COW_SIZE + CHILD_INFO_TYPE_MODULE_COW_SIZE, + CHILD_INFO_TYPE_REPL_OUTPUT_BYTES } childInfoType; struct valkeyServer { @@ -2102,6 +2103,8 @@ struct valkeyServer { mstime_t busy_reply_threshold; /* Script / module timeout in milliseconds */ int pre_command_oom_state; /* OOM before command (script?) was started */ int script_disable_deny_script; /* Allow running commands marked "noscript" inside a script. */ + int lua_enable_insecure_api; /* Config to enable insecure api */ + int lua_insecure_api_current; /* Current value of if insecure apis are enabled, used to determine if flush is needed. */ /* Lazy free */ int lazyfree_lazy_eviction; int lazyfree_lazy_expire; @@ -3008,7 +3011,7 @@ int aofRewriteLimited(void); /* Child info */ void openChildInfoPipe(void); void closeChildInfoPipe(void); -void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress, char *pname); +void sendChildInfoGeneric(childInfoType info_type, size_t keys, size_t repl_output_bytes, double progress, char *pname); void sendChildCowInfo(childInfoType info_type, char *pname); void sendChildInfo(childInfoType info_type, size_t keys, char *pname); void receiveChildInfo(void); diff --git a/src/unit/test_files.h b/src/unit/test_files.h index 7ef42439eb..f182018c6f 100644 --- a/src/unit/test_files.h +++ b/src/unit/test_files.h @@ -50,6 +50,7 @@ int test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyHashtable(int argc, char **arg int test_kvstoreIteratorRemoveAllKeysDeleteEmptyHashtable(int argc, char **argv, int flags); int test_kvstoreHashtableIteratorRemoveAllKeysNoDeleteEmptyHashtable(int argc, char **argv, int flags); int test_kvstoreHashtableIteratorRemoveAllKeysDeleteEmptyHashtable(int argc, char **argv, int flags); +int test_kvstoreHashtableExpand(int argc, char **argv, int flags); int test_listpackCreateIntList(int argc, char **argv, int flags); int test_listpackCreateList(int argc, char **argv, int flags); int test_listpackLpPrepend(int argc, char **argv, int flags); @@ -240,7 +241,7 @@ unitTest __test_dict_c[] = {{"test_dictCreate", test_dictCreate}, {"test_dictAdd unitTest __test_endianconv_c[] = {{"test_endianconv", test_endianconv}, {NULL, NULL}}; unitTest __test_hashtable_c[] = {{"test_cursor", test_cursor}, {"test_set_hash_function_seed", test_set_hash_function_seed}, {"test_add_find_delete", test_add_find_delete}, {"test_add_find_delete_avoid_resize", test_add_find_delete_avoid_resize}, {"test_instant_rehashing", test_instant_rehashing}, {"test_bucket_chain_length", test_bucket_chain_length}, {"test_two_phase_insert_and_pop", test_two_phase_insert_and_pop}, {"test_replace_reallocated_entry", test_replace_reallocated_entry}, {"test_incremental_find", test_incremental_find}, {"test_scan", test_scan}, {"test_iterator", test_iterator}, {"test_safe_iterator", test_safe_iterator}, {"test_compact_bucket_chain", test_compact_bucket_chain}, {"test_random_entry", test_random_entry}, {"test_random_entry_with_long_chain", test_random_entry_with_long_chain}, {"test_random_entry_sparse_table", test_random_entry_sparse_table}, {"test_all_memory_freed", test_all_memory_freed}, {NULL, NULL}}; unitTest __test_intset_c[] = {{"test_intsetValueEncodings", test_intsetValueEncodings}, {"test_intsetBasicAdding", test_intsetBasicAdding}, {"test_intsetLargeNumberRandomAdd", test_intsetLargeNumberRandomAdd}, {"test_intsetUpgradeFromint16Toint32", test_intsetUpgradeFromint16Toint32}, {"test_intsetUpgradeFromint16Toint64", test_intsetUpgradeFromint16Toint64}, {"test_intsetUpgradeFromint32Toint64", test_intsetUpgradeFromint32Toint64}, {"test_intsetStressLookups", test_intsetStressLookups}, {"test_intsetStressAddDelete", test_intsetStressAddDelete}, {NULL, NULL}}; -unitTest __test_kvstore_c[] = {{"test_kvstoreAdd16Keys", test_kvstoreAdd16Keys}, {"test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyHashtable", test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyHashtable}, {"test_kvstoreIteratorRemoveAllKeysDeleteEmptyHashtable", test_kvstoreIteratorRemoveAllKeysDeleteEmptyHashtable}, {"test_kvstoreHashtableIteratorRemoveAllKeysNoDeleteEmptyHashtable", test_kvstoreHashtableIteratorRemoveAllKeysNoDeleteEmptyHashtable}, {"test_kvstoreHashtableIteratorRemoveAllKeysDeleteEmptyHashtable", test_kvstoreHashtableIteratorRemoveAllKeysDeleteEmptyHashtable}, {NULL, NULL}}; +unitTest __test_kvstore_c[] = {{"test_kvstoreAdd16Keys", test_kvstoreAdd16Keys}, {"test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyHashtable", test_kvstoreIteratorRemoveAllKeysNoDeleteEmptyHashtable}, {"test_kvstoreIteratorRemoveAllKeysDeleteEmptyHashtable", test_kvstoreIteratorRemoveAllKeysDeleteEmptyHashtable}, {"test_kvstoreHashtableIteratorRemoveAllKeysNoDeleteEmptyHashtable", test_kvstoreHashtableIteratorRemoveAllKeysNoDeleteEmptyHashtable}, {"test_kvstoreHashtableIteratorRemoveAllKeysDeleteEmptyHashtable", test_kvstoreHashtableIteratorRemoveAllKeysDeleteEmptyHashtable}, {"test_kvstoreHashtableExpand", test_kvstoreHashtableExpand}, {NULL, NULL}}; unitTest __test_listpack_c[] = {{"test_listpackCreateIntList", test_listpackCreateIntList}, {"test_listpackCreateList", test_listpackCreateList}, {"test_listpackLpPrepend", test_listpackLpPrepend}, {"test_listpackLpPrependInteger", test_listpackLpPrependInteger}, {"test_listpackGetELementAtIndex", test_listpackGetELementAtIndex}, {"test_listpackPop", test_listpackPop}, {"test_listpackGetELementAtIndex2", test_listpackGetELementAtIndex2}, {"test_listpackIterate0toEnd", test_listpackIterate0toEnd}, {"test_listpackIterate1toEnd", test_listpackIterate1toEnd}, {"test_listpackIterate2toEnd", test_listpackIterate2toEnd}, {"test_listpackIterateBackToFront", test_listpackIterateBackToFront}, {"test_listpackIterateBackToFrontWithDelete", test_listpackIterateBackToFrontWithDelete}, {"test_listpackDeleteWhenNumIsMinusOne", test_listpackDeleteWhenNumIsMinusOne}, {"test_listpackDeleteWithNegativeIndex", test_listpackDeleteWithNegativeIndex}, {"test_listpackDeleteInclusiveRange0_0", test_listpackDeleteInclusiveRange0_0}, {"test_listpackDeleteInclusiveRange0_1", test_listpackDeleteInclusiveRange0_1}, {"test_listpackDeleteInclusiveRange1_2", test_listpackDeleteInclusiveRange1_2}, {"test_listpackDeleteWitStartIndexOutOfRange", test_listpackDeleteWitStartIndexOutOfRange}, {"test_listpackDeleteWitNumOverflow", test_listpackDeleteWitNumOverflow}, {"test_listpackBatchDelete", test_listpackBatchDelete}, {"test_listpackDeleteFooWhileIterating", test_listpackDeleteFooWhileIterating}, {"test_listpackReplaceWithSameSize", test_listpackReplaceWithSameSize}, {"test_listpackReplaceWithDifferentSize", test_listpackReplaceWithDifferentSize}, {"test_listpackRegressionGt255Bytes", test_listpackRegressionGt255Bytes}, {"test_listpackCreateLongListAndCheckIndices", test_listpackCreateLongListAndCheckIndices}, {"test_listpackCompareStrsWithLpEntries", test_listpackCompareStrsWithLpEntries}, {"test_listpackLpMergeEmptyLps", test_listpackLpMergeEmptyLps}, {"test_listpackLpMergeLp1Larger", test_listpackLpMergeLp1Larger}, {"test_listpackLpMergeLp2Larger", test_listpackLpMergeLp2Larger}, {"test_listpackLpNextRandom", test_listpackLpNextRandom}, {"test_listpackLpNextRandomCC", test_listpackLpNextRandomCC}, {"test_listpackRandomPairWithOneElement", test_listpackRandomPairWithOneElement}, {"test_listpackRandomPairWithManyElements", test_listpackRandomPairWithManyElements}, {"test_listpackRandomPairsWithOneElement", test_listpackRandomPairsWithOneElement}, {"test_listpackRandomPairsWithManyElements", test_listpackRandomPairsWithManyElements}, {"test_listpackRandomPairsUniqueWithOneElement", test_listpackRandomPairsUniqueWithOneElement}, {"test_listpackRandomPairsUniqueWithManyElements", test_listpackRandomPairsUniqueWithManyElements}, {"test_listpackPushVariousEncodings", test_listpackPushVariousEncodings}, {"test_listpackLpFind", test_listpackLpFind}, {"test_listpackLpValidateIntegrity", test_listpackLpValidateIntegrity}, {"test_listpackNumberOfElementsExceedsLP_HDR_NUMELE_UNKNOWN", test_listpackNumberOfElementsExceedsLP_HDR_NUMELE_UNKNOWN}, {"test_listpackStressWithRandom", test_listpackStressWithRandom}, {"test_listpackSTressWithVariableSize", test_listpackSTressWithVariableSize}, {"test_listpackBenchmarkInit", test_listpackBenchmarkInit}, {"test_listpackBenchmarkLpAppend", test_listpackBenchmarkLpAppend}, {"test_listpackBenchmarkLpFindString", test_listpackBenchmarkLpFindString}, {"test_listpackBenchmarkLpFindNumber", test_listpackBenchmarkLpFindNumber}, {"test_listpackBenchmarkLpSeek", test_listpackBenchmarkLpSeek}, {"test_listpackBenchmarkLpValidateIntegrity", test_listpackBenchmarkLpValidateIntegrity}, {"test_listpackBenchmarkLpCompareWithString", test_listpackBenchmarkLpCompareWithString}, {"test_listpackBenchmarkLpCompareWithNumber", test_listpackBenchmarkLpCompareWithNumber}, {"test_listpackBenchmarkFree", test_listpackBenchmarkFree}, {NULL, NULL}}; unitTest __test_networking_c[] = {{"test_writeToReplica", test_writeToReplica}, {"test_postWriteToReplica", test_postWriteToReplica}, {"test_backupAndUpdateClientArgv", test_backupAndUpdateClientArgv}, {"test_rewriteClientCommandArgument", test_rewriteClientCommandArgument}, {NULL, NULL}}; unitTest __test_object_c[] = {{"test_object_with_key", test_object_with_key}, {NULL, NULL}}; diff --git a/src/unit/test_kvstore.c b/src/unit/test_kvstore.c index ac4273214c..0d98b7b9de 100644 --- a/src/unit/test_kvstore.c +++ b/src/unit/test_kvstore.c @@ -223,3 +223,20 @@ int test_kvstoreHashtableIteratorRemoveAllKeysDeleteEmptyHashtable(int argc, cha kvstoreRelease(kvs2); return 0; } + +int test_kvstoreHashtableExpand(int argc, char **argv, int flags) { + UNUSED(argc); + UNUSED(argv); + UNUSED(flags); + + kvstore *kvs = kvstoreCreate(&KvstoreHashtableTestType, 0, KVSTORE_ALLOCATE_HASHTABLES_ON_DEMAND | KVSTORE_FREE_EMPTY_HASHTABLES); + + TEST_ASSERT(kvstoreGetHashtable(kvs, 0) == NULL); + TEST_ASSERT(kvstoreHashtableExpand(kvs, 0, 10000)); + TEST_ASSERT(kvstoreGetHashtable(kvs, 0) != NULL); + TEST_ASSERT(kvstoreBuckets(kvs) > 0); + TEST_ASSERT(kvstoreBuckets(kvs) == kvstoreHashtableBuckets(kvs, 0)); + + kvstoreRelease(kvs); + return 0; +} diff --git a/src/version.h b/src/version.h index 8fd956e9ad..904608b0f1 100644 --- a/src/version.h +++ b/src/version.h @@ -4,8 +4,8 @@ * similar. */ #define SERVER_NAME "valkey" #define SERVER_TITLE "Valkey" -#define VALKEY_VERSION "8.1.3" -#define VALKEY_VERSION_NUM 0x00080103 +#define VALKEY_VERSION "8.1.4" +#define VALKEY_VERSION_NUM 0x00080104 /* The release stage is used in order to provide release status information. * In unstable branch the status is always "dev". * During release process the status will be set to rc1,rc2...rcN. diff --git a/tests/assets/divergent-shard-1.conf b/tests/assets/divergent-shard-1.conf new file mode 100644 index 0000000000..6f347d0470 --- /dev/null +++ b/tests/assets/divergent-shard-1.conf @@ -0,0 +1,3 @@ +43ee1cacd6948ee96bb367eb8795e62e8d153f05 127.0.0.1:0@6379,,tls-port=0,shard-id=f91532eb722943e035f34292cf586f3f750d65bd myself,master - 0 1749488968682 13 connected 0-16383 +8d89f4d4e7c57a2819277732f86213241c3ec0d3 127.0.0.1:0@6380,,tls-port=0,shard-id=a91532eb722943e035f34292cf586f3f750d65bd slave 43ee1cacd6948ee96bb367eb8795e62e8d153f05 0 1749488968682 13 connected +vars currentEpoch 13 lastVoteEpoch 9 \ No newline at end of file diff --git a/tests/assets/divergent-shard-2.conf b/tests/assets/divergent-shard-2.conf new file mode 100644 index 0000000000..3f7d75c947 --- /dev/null +++ b/tests/assets/divergent-shard-2.conf @@ -0,0 +1,3 @@ +8d89f4d4e7c57a2819277732f86213241c3ec0d3 127.0.0.1:0@6380,,tls-port=0,shard-id=a91532eb722943e035f34292cf586f3f750d65bd slave 43ee1cacd6948ee96bb367eb8795e62e8d153f05 0 1749488968682 13 connected +43ee1cacd6948ee96bb367eb8795e62e8d153f05 127.0.0.1:0@6379,,tls-port=0,shard-id=f91532eb722943e035f34292cf586f3f750d65bd myself,master - 0 1749488968682 13 connected 0-16383 +vars currentEpoch 13 lastVoteEpoch 9 \ No newline at end of file diff --git a/tests/assets/divergent-shard-3.conf b/tests/assets/divergent-shard-3.conf new file mode 100644 index 0000000000..363837f436 --- /dev/null +++ b/tests/assets/divergent-shard-3.conf @@ -0,0 +1,3 @@ +43ee1cacd6948ee96bb367eb8795e62e8d153f05 127.0.0.1:0@6379,,tls-port=0,shard-id=f91532eb722943e035f34292cf586f3f750d65bd master - 0 1749488968682 13 connected 0-16383 +8d89f4d4e7c57a2819277732f86213241c3ec0d3 127.0.0.1:0@6380,,tls-port=0,shard-id=a91532eb722943e035f34292cf586f3f750d65bd myself,slave 43ee1cacd6948ee96bb367eb8795e62e8d153f05 0 1749488968682 13 connected +vars currentEpoch 13 lastVoteEpoch 9 \ No newline at end of file diff --git a/tests/assets/divergent-shard-4.conf b/tests/assets/divergent-shard-4.conf new file mode 100644 index 0000000000..5fcd97bfad --- /dev/null +++ b/tests/assets/divergent-shard-4.conf @@ -0,0 +1,3 @@ +8d89f4d4e7c57a2819277732f86213241c3ec0d3 127.0.0.1:0@6380,,tls-port=0,shard-id=a91532eb722943e035f34292cf586f3f750d65bd myself,slave 43ee1cacd6948ee96bb367eb8795e62e8d153f05 0 1749488968682 13 connected +43ee1cacd6948ee96bb367eb8795e62e8d153f05 127.0.0.1:0@6379,,tls-port=0,shard-id=f91532eb722943e035f34292cf586f3f750d65bd master - 0 1749488968682 13 connected 0-16383 +vars currentEpoch 13 lastVoteEpoch 9 \ No newline at end of file diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index 02df4bb4f8..d65ff82f2e 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -689,9 +689,10 @@ tags {"aof cluster external:skip"} { append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" } - start_server_aof [list dir $server_path cluster-enabled yes] { + start_server_aof [list dir $server_path cluster-enabled yes cluster-port [find_available_port $::baseport $::portcount]] { assert_equal [r ping] {PONG} } + clean_aof_persistence $aof_dirpath } test {Test command check in aof won't crash} { diff --git a/tests/integration/dual-channel-replication.tcl b/tests/integration/dual-channel-replication.tcl index b49b614cd1..49d0c39139 100644 --- a/tests/integration/dual-channel-replication.tcl +++ b/tests/integration/dual-channel-replication.tcl @@ -95,6 +95,48 @@ start_server {tags {"dual-channel-replication external:skip"}} { fail "Replicas and primary offsets were unable to match." } } + + test "Dual-channel replication counts snapshot bytes" { + wait_for_condition 50 100 { + [getInfoProperty [$primary info stats] "total_net_repl_output_bytes"] > 0 + } else { + fail "Replication output bytes not updated" + } + } + } +} + +start_server {tags {"dual-channel-replication external:skip"}} { + set replica [srv 0 client] + set replica_host [srv 0 host] + set replica_port [srv 0 port] + start_server {} { + set primary [srv 0 client] + set primary_host [srv 0 host] + set primary_port [srv 0 port] + + $primary config set repl-diskless-sync yes + $primary config set repl-diskless-sync-delay 0 + $primary config set dual-channel-replication-enabled yes + + $replica config set repl-diskless-sync yes + $replica config set repl-diskless-load swapdb + $replica config set dual-channel-replication-enabled yes + + for {set j 0} {$j < 100} {incr j} { + $primary set key$j [string repeat x 100] + } + $primary config resetstat + + test "dual-channel replication reports rdb transfer bytes" { + $replica replicaof $primary_host $primary_port + verify_replica_online $primary 0 700 + wait_for_condition 50 100 { + [getInfoProperty [$primary info stats] "total_net_repl_output_bytes"] > 1000 + } else { + fail "Replication output bytes not updated" + } + } } } diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 662449134a..9b6fa13c84 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -264,6 +264,30 @@ proc CI {index field} { getInfoProperty [R $index cluster info] $field } +# Provide easy access to CLIENT INFO properties from CLIENT INFO string. +proc get_field_in_client_info {info field} { + set info [string trim $info] + foreach item [split $info " "] { + set kv [split $item "="] + set k [lindex $kv 0] + if {[string match $field $k]} { + return [lindex $kv 1] + } + } + return "" +} + +# Provide easy access to CLIENT INFO properties from CLIENT LIST string. +proc get_field_in_client_list {id client_list filed} { + set list [split $client_list "\r\n"] + foreach info $list { + if {[string match "id=$id *" $info] } { + return [get_field_in_client_info $info $filed] + } + } + return "" +} + # Test wrapped into run_solo are sent back from the client to the # test server, so that the test server will send them again to # clients once the clients are idle. @@ -280,6 +304,7 @@ proc cleanup {} { if {!$::quiet} {puts -nonewline "Cleanup: may take some time... "} flush stdout catch {exec rm -rf {*}[glob tests/tmp/valkey.conf.*]} + catch {exec rm -rf {*}[glob tests/tmp/nodes.conf.*]} catch {exec rm -rf {*}[glob tests/tmp/server*.*]} catch {exec rm -rf {*}[glob tests/tmp/*.acl.*]} if {!$::quiet} {puts "OK"} diff --git a/tests/unit/cluster/divergent-cluster-shardid-conf.tcl b/tests/unit/cluster/divergent-cluster-shardid-conf.tcl new file mode 100644 index 0000000000..6885eac214 --- /dev/null +++ b/tests/unit/cluster/divergent-cluster-shardid-conf.tcl @@ -0,0 +1,17 @@ +tags {external:skip cluster singledb} { + set old_singledb $::singledb + set ::singledb 1 + # Start a cluster with a divergent shard ID configuration + test "divergent cluster shardid conflict" { + for {set i 1} {$i <= 4} {incr i} { + if {$::verbose} { puts "Testing for tests/assets/divergent-shard-$i.conf"; flush stdout;} + exec cp -f tests/assets/divergent-shard-$i.conf tests/tmp/nodes.conf.divergent + start_server {overrides {"cluster-enabled" "yes" "cluster-config-file" "../nodes.conf.divergent"}} { + set shardid [r CLUSTER MYSHARDID] + set count [exec grep -c $shardid tests/tmp/nodes.conf.divergent]; + assert_equal $count 2 "Expect shard ID to be present twice in the configuration file" + } + } + } + set ::singledb $old_singledb +} diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 3736538105..5b7e8759b9 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -951,6 +951,43 @@ start_server {tags {"expire"}} { } } +start_server {tags {expire} overrides {hz 100}} { + test {Active expiration triggers hashtable shrink} { + set persistent_keys 5 + set volatile_keys 100 + set total_keys [expr $persistent_keys + $volatile_keys] + + for {set i 0} {$i < $persistent_keys} {incr i} { + r set "key_$i" "value_$i" + } + for {set i 0} {$i < $volatile_keys} {incr i} { + r psetex "expire_key_${i}" 100 "expire_value_${i}" + } + set table_size_before_expire [main_hash_table_size] + + # Verify keys are set + assert_equal $total_keys [r dbsize] + + # Wait for active expiration + wait_for_condition 100 50 { + [r dbsize] eq $persistent_keys + } else { + fail "Keys not expired" + } + + # Wait for the table to shrink and active rehashing finish + wait_for_condition 100 50 { + [main_hash_table_size] < $table_size_before_expire + } else { + puts [r debug htstats 9] + fail "Table didn't shrink" + } + + # Verify server is still responsive + assert_equal [r ping] {PONG} + } {} {needs:debug} +} + start_cluster 1 0 {tags {"expire external:skip cluster"}} { test "expire scan should skip dictionaries with lot's of empty buckets" { r debug set-active-expire 0 diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 5fe6a18d46..9923141034 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -139,28 +139,6 @@ start_server {tags {"introspection"}} { assert_error "ERR *greater than 0*" {r client list maxage -1} } - proc get_field_in_client_info {info field} { - set info [string trim $info] - foreach item [split $info " "] { - set kv [split $item "="] - set k [lindex $kv 0] - if {[string match $field $k]} { - return [lindex $kv 1] - } - } - return "" - } - - proc get_field_in_client_list {id client_list filed} { - set list [split $client_list "\r\n"] - foreach info $list { - if {[string match "id=$id *" $info] } { - return [get_field_in_client_info $info $filed] - } - } - return "" - } - proc get_client_tot_in_out_cmds {id} { set info_list [r client list] set in [get_field_in_client_list $id $info_list "tot-net-in"] diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index b075a11fea..3ebe685064 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -59,6 +59,15 @@ run_solo {defrag} { return [list $key $field] } + # Make a deferring client with CLIENT REPLY OFF sync with the server by + # temporarily setting CLIENT REPLY ON and waiting for the reply, then + # switching back to CLIENT REPLY OFF. + proc client_reply_off_wait_for_server {rd} { + $rd client reply on + assert_equal OK [$rd read] + $rd client reply off + } + # Logs defragging state if ::verbose is true proc log_frag {title} { # Note, this delay is outside of the "if" so that behavior is the same, with and @@ -238,27 +247,23 @@ run_solo {defrag} { perform_defrag_test $title populate { # add a mass of string keys set rd [valkey_deferring_client] + $rd client reply off for {set j 0} {$j < $n} {incr j} { $rd setrange $j 250 a if {$j % 3 == 0} { $rd expire $j 1000 ;# put expiration on some } - } - for {set j 0} {$j < $n} {incr j} { - $rd read ; # Discard replies - if {$j % 3 == 0} { - $rd read - } + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} } assert {[scan [regexp -inline {expires\=([\d]*)} [r info keyspace]] expires=%d] > 0} } fragment { # delete half of the keys for {set j 0} {$j < $n} {incr j 2} { $rd del $j + if {$j % 1000 == 998} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j 2} { $rd read } ; # Discard replies $rd close - + # use custom slower defrag speed to start so that while_defragging has time r config set active-defrag-cycle-min 2 r config set active-defrag-cycle-max 3 @@ -324,19 +329,19 @@ run_solo {defrag} { # Populate memory with interleaving script-key pattern of same size set dummy_script "--[string repeat x 450]\nreturn " set rd [valkey_deferring_client] + $rd client reply off for {set j 0} {$j < $n} {incr j} { set val "$dummy_script[format "%06d" $j]" $rd script load $val $rd set k$j $val - } - for {set j 0} {$j < $n} {incr j} { - $rd read ; # Discard script load replies - $rd read ; # Discard set replies + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} } } fragment { # Delete all the keys to create fragmentation - for {set j 0} {$j < $n} {incr j} { $rd del k$j } - for {set j 0} {$j < $n} {incr j} { $rd read } ; # Discard del replies + for {set j 0} {$j < $n} {incr j} { + $rd del k$j + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} + } $rd close } } @@ -350,22 +355,23 @@ run_solo {defrag} { perform_defrag_test $title populate { set rd [valkey_deferring_client] + $rd client reply off set val [string repeat A 250] set k 0 set f 0 for {set j 0} {$j < $n} {incr j} { $rd hset k$k f$f $val lassign [next_exp_kf $k $f] k f + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j} { $rd read } ; # Discard replies } fragment { set k 0 set f 0 for {set j 0} {$j < $n} {incr j 2} { $rd hdel k$k f$f lassign [next_exp_kf $k $f 2] k f + if {$j % 1000 == 998} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j 2} { $rd read } ; # Discard replies $rd close } } @@ -380,14 +386,15 @@ run_solo {defrag} { perform_defrag_test $title populate { set rd [valkey_deferring_client] + $rd client reply off set val [string repeat A 350] set k 0 set f 0 for {set j 0} {$j < $n} {incr j} { $rd lpush k$k $val lassign [next_exp_kf $k $f] k f + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j} { $rd read } ; # Discard replies } fragment { set k 0 set f 0 @@ -395,8 +402,8 @@ run_solo {defrag} { $rd ltrim k$k 1 -1 ;# deletes the leftmost item $rd lmove k$k k$k LEFT RIGHT ;# rotates the leftmost item to the right side lassign [next_exp_kf $k $f 2] k f + if {$j % 1000 == 998} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j 2} { $rd read; $rd read } ; # Discard replies $rd close } } @@ -410,22 +417,23 @@ run_solo {defrag} { perform_defrag_test $title populate { set rd [valkey_deferring_client] + $rd client reply off set val [string repeat A 300] set k 0 set f 0 for {set j 0} {$j < $n} {incr j} { $rd sadd k$k $val$f lassign [next_exp_kf $k $f] k f + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j} { $rd read } ; # Discard replies } fragment { set k 0 set f 0 for {set j 0} {$j < $n} {incr j 2} { $rd srem k$k $val$f lassign [next_exp_kf $k $f 2] k f + if {$j % 1000 == 998} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j 2} { $rd read } ; # Discard replies $rd close } } @@ -439,22 +447,23 @@ run_solo {defrag} { perform_defrag_test $title populate { set rd [valkey_deferring_client] + $rd client reply off set val [string repeat A 250] set k 0 set f 0 for {set j 0} {$j < $n} {incr j} { $rd zadd k$k [expr rand()] $val$f lassign [next_exp_kf $k $f] k f + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j} { $rd read } ; # Discard replies } fragment { set k 0 set f 0 for {set j 0} {$j < $n} {incr j 2} { $rd zrem k$k $val$f lassign [next_exp_kf $k $f 2] k f + if {$j % 1000 == 998} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j 2} { $rd read } ; # Discard replies $rd close } } @@ -469,16 +478,17 @@ run_solo {defrag} { perform_defrag_test $title populate { set rd [valkey_deferring_client] + $rd client reply off set val [string repeat A 50] for {set j 0} {$j < $n} {incr j} { $rd xadd k$j * field1 $val field2 $val + if {$j % 1000 == 999} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j} { $rd read } ; # Discard replies } fragment { for {set j 0} {$j < $n} {incr j 2} { $rd del k$j + if {$j % 1000 == 998} {client_reply_off_wait_for_server $rd} } - for {set j 0} {$j < $n} {incr j 2} { $rd read } ; # Discard replies $rd close } } diff --git a/tests/unit/moduleapi/datatype.tcl b/tests/unit/moduleapi/datatype.tcl index d83fd00da8..28bbf5c0f2 100644 --- a/tests/unit/moduleapi/datatype.tcl +++ b/tests/unit/moduleapi/datatype.tcl @@ -131,4 +131,11 @@ start_server {tags {"modules"}} { assert_equal 1 [llength $keys] } + + test {DataType: check memory usage} { + r flushdb + set large_key [string repeat A 100000] + r datatype.set $large_key 111 bar + assert_morethan [r memory usage $large_key] 100000 + } } diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 174609bdd7..fc5774fdef 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -900,6 +900,7 @@ start_server {tags {"multi"}} { r watch b{t} a{t} r flushall r ping + r unwatch } test "AUTH errored inside MULTI will add the reply" { diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 98b623a39f..53aebe64eb 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -618,13 +618,91 @@ start_server {tags {"scripting"}} { assert_error {NOSCRIPT*} {r evalsha fd758d1589d044dd850a6f05d52f2eefd27f033f 1 mykey} } + + test {EVAL - Test table unpack with invalid indexes} { + catch {r eval { return {unpack({1,2,3}, -2, 2147483647)} } 0} e + assert_match {*too many results to unpack*} $e + catch {r eval { return {unpack({1,2,3}, 0, 2147483647)} } 0} e + assert_match {*too many results to unpack*} $e + catch {r eval { return {unpack({1,2,3}, -2147483648, -2)} } 0} e + assert_match {*too many results to unpack*} $e + set res [r eval { return {unpack({1,2,3}, -1, -2)} } 0] + assert_match {} $res + set res [r eval { return {unpack({1,2,3}, 1, -1)} } 0] + assert_match {} $res + + # unpack with range -1 to 5, verify nil indexes + set res [r eval { + local function unpack_to_list(t, i, j) + local n, v = select('#', unpack(t, i, j)), {unpack(t, i, j)} + for i = 1, n do v[i] = v[i] or '_NIL_' end + v.n = n + return v + end + + return unpack_to_list({1,2,3}, -1, 5) + } 0] + assert_match {_NIL_ _NIL_ 1 2 3 _NIL_ _NIL_} $res + + # unpack with negative range, verify nil indexes + set res [r eval { + local function unpack_to_list(t, i, j) + local n, v = select('#', unpack(t, i, j)), {unpack(t, i, j)} + for i = 1, n do v[i] = v[i] or '_NIL_' end + v.n = n + return v + end + + return unpack_to_list({1,2,3}, -2147483648, -2147483646) + } 0] + assert_match {_NIL_ _NIL_ _NIL_} $res + } {} + + test "Try trick readonly table on basic types metatable" { + # Run the following scripts for basic types. Either getmetatable() + # should return nil or the metatable must be readonly. + set scripts { + {getmetatable(nil).__index = function() return 1 end} + {getmetatable('').__index = function() return 1 end} + {getmetatable(123.222).__index = function() return 1 end} + {getmetatable(true).__index = function() return 1 end} + {getmetatable(function() return 1 end).__index = function() return 1 end} + {getmetatable(coroutine.create(function() return 1 end)).__index = function() return 1 end} + } + + foreach code $scripts { + catch {r eval $code 0} e + assert { + [string match "*attempt to index a nil value*" $e] || + [string match "*Attempt to modify a readonly table*" $e] + } + } + } + + test {Dynamic reset of lua engine with insecure API config change} { + # Ensure insecure API is not available by default + assert_error {*Script attempted to access nonexistent global variable 'getfenv'*} { + r eval "return getfenv()" 0 + } + + # Verify that enabling the config `lua-enable-insecure-api` allows insecure API access + r config set lua-enable-insecure-api yes + assert_equal {} [r eval "return getfenv()" 0] + + r config set lua-enable-insecure-api no + assert_error {*Script attempted to access nonexistent global variable 'getfenv'*} { + r eval "return getfenv()" 0 + } + } {} {external:skip} + test {SCRIPTING FLUSH ASYNC} { + r script flush sync for {set j 0} {$j < 100} {incr j} { r script load "return $j" } - assert { [string match "*number_of_cached_scripts:100*" [r info Memory]] } + assert_match "*number_of_cached_scripts:100*" [r info Memory] r script flush async - assert { [string match "*number_of_cached_scripts:0*" [r info Memory]] } + assert_match "*number_of_cached_scripts:0*" [r info Memory] } test {SCRIPT EXISTS - can detect already defined scripts?} { @@ -1155,6 +1233,45 @@ start_server {tags {"scripting"}} { } {*Script attempted to access nonexistent global variable 'print'*} } +# start a new server to test the large-memory tests +start_server {tags {"scripting external:skip large-memory"}} { + test {EVAL - Test long escape sequences for strings} { + r eval { + -- Generate 1gb '==...==' separator + local s = string.rep('=', 1024 * 1024) + local t = {} for i=1,1024 do t[i] = s end + local sep = table.concat(t) + collectgarbage('collect') + + local code = table.concat({'return [',sep,'[x]',sep,']'}) + collectgarbage('collect') + + -- Load the code and run it. Script will return the string length. + -- Escape sequence: [=....=[ to ]=...=] will be ignored + -- Actual string is a single character: 'x'. Script will return 1 + local func = loadstring(code) + return #func() + } 0 + } {1} + + test {EVAL - Lua can parse string with too many new lines} { + # Create a long string consisting only of newline characters. When Lua + # fails to parse a string, it typically includes a snippet like + # "... near ..." in the error message to indicate the last recognizable + # token. In this test, since the input contains only newlines, there + # should be no identifiable token, so the error message should contain + # only the actual error, without a near clause. + + r eval { + local s = string.rep('\n', 1024 * 1024) + local t = {} for i=1,2048 do t[#t+1] = s end + local lines = table.concat(t) + local fn, err = loadstring(lines) + return err + } 0 + } {*chunk has too many lines} +} + # Start a new server since the last test in this stanza will kill the # instance at all. start_server {tags {"scripting"}} { @@ -2464,4 +2581,21 @@ start_server {tags {"scripting"}} { # Using a null byte never seemed to work with functions, so # we don't have a test for that case. } + + test {EVAL - explicit error() call handling} { + # error("simple string error") + assert_error {ERR user_script:1: simple string error script: *} { + r eval "error('simple string error')" 0 + } + + # error({"err": "ERR table error"}) + assert_error {ERR table error script: *} { + r eval "error({err='ERR table error'})" 0 + } + + # error({}) + assert_error {ERR unknown error script: *} { + r eval "error({})" 0 + } + } } diff --git a/tests/unit/tracking.tcl b/tests/unit/tracking.tcl index ec9368b4f4..4edc8573cb 100644 --- a/tests/unit/tracking.tcl +++ b/tests/unit/tracking.tcl @@ -38,6 +38,34 @@ start_server {tags {"tracking network logreqres:skip"}} { } } + test {Client tracking prefixes memory overhead} { + r CLIENT TRACKING off + set tot_mem_before [get_field_in_client_info [r client info] "tot-mem"] + + # We add multiple $i to prefix to avoid prefix conflicts, so in this + # args we will have about 20000 rax nodes. + set args {} + for {set i 0} {$i < 10240} {incr i} { + lappend args PREFIX + lappend args PREFIX-$i-$i-$i-$i-$i-$i-$i + } + r CLIENT TRACKING on BCAST {*}$args + + set arch_bits [s arch_bits] + set tot_mem_after [get_field_in_client_info [r client info] "tot-mem"] + set diff [expr $tot_mem_after - $tot_mem_before] + + # In 64 bits, before we would consume about 20000 * (4 * 8), that is 640000. + # And now we are 20000 * (4 + 8), that is 240000. + if {$arch_bits == 64} { + assert_lessthan $diff 300000 + } elseif {$arch_bits == 32} { + assert_lessthan $diff 200000 + } + + r CLIENT TRACKING off + } + test {Clients are able to enable tracking and redirect it} { r CLIENT TRACKING on REDIRECT $redir_id } {*OK} diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 296b6cf9bf..c8cb406b0a 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -2453,4 +2453,36 @@ foreach {pop} {BLPOP BLMPOP_RIGHT} { $rd close } + test "CLIENT NO-TOUCH with BRPOP and RPUSH regression test" { + # Test scenario: + # 1. Client 1: CLIENT NO-TOUCH on + # 2. Client 2: BRPOP mylist 0 + # 3. Client 1: RPUSH mylist elem + + # cleanup first + r del mylist + + # Create two test clients + set rd1 [valkey_deferring_client] + set rd2 [valkey_deferring_client] + + # Client 1: Enable CLIENT NO-TOUCH + $rd1 client no-touch on + assert_equal {OK} [$rd1 read] + + # Client 2: Block waiting for elements in mylist + $rd2 brpop mylist 0 + wait_for_blocked_client + + # Client 1: Push an element to mylist + $rd1 rpush mylist elem + assert_equal {1} [$rd1 read] + + # Verify Client 2 received the element + assert_equal {mylist elem} [$rd2 read] + + $rd1 close + $rd2 close + } + } ;# stop servers diff --git a/tests/unit/type/string.tcl b/tests/unit/type/string.tcl index 9bd04fdd66..3141cc4adc 100644 --- a/tests/unit/type/string.tcl +++ b/tests/unit/type/string.tcl @@ -59,6 +59,31 @@ start_server {tags {"string"}} { } {10000} } + test {memoryusage of string} { + # Simple test + set key "key" + set value "value" + r set $key $value + assert_lessthan_equal [expr [string length $key] + [string length $value]] [r memory usage key] + + # Big value + set key "key" + set value [string repeat A 100000] + r set $key $value + assert_lessthan_equal [expr [string length $key] + [string length $value]] [r memory usage key] + + # Big key + set key [string repeat A 100000] + set value "value" + r set $key $value + assert_lessthan_equal [expr [string length $key] + [string length $value]] [r memory usage key] + + # Big key for int + set key [string repeat B 100000] + r incr $key + assert_lessthan_equal [string length $key] [r memory usage $key] + } + test "SETNX target key missing" { r del novar assert_equal 1 [r setnx novar foobared]