From 0d8345310f9b755b7dc8b7b2ebb7281d5f47c432 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Mon, 11 Aug 2025 14:50:01 +0200 Subject: [PATCH 01/20] core implementation --- crates/consistent-hashing/Cargo.toml | 17 ++ crates/consistent-hashing/README.md | 60 ++++++ crates/consistent-hashing/src/lib.rs | 292 +++++++++++++++++++++++++++ 3 files changed, 369 insertions(+) create mode 100644 crates/consistent-hashing/Cargo.toml create mode 100644 crates/consistent-hashing/README.md create mode 100644 crates/consistent-hashing/src/lib.rs diff --git a/crates/consistent-hashing/Cargo.toml b/crates/consistent-hashing/Cargo.toml new file mode 100644 index 0000000..53f4e02 --- /dev/null +++ b/crates/consistent-hashing/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "consistent-hashing" +version = "0.1.0" +edition = "2021" +description = "Constant time consistent hashing algorithms." +repository = "https://github.com/github/rust-gems" +license = "MIT" +keywords = ["probabilistic", "algorithm", "consistent hashing", "jump hashing", "rendezvous hashing"] +categories = ["algorithms", "data-structures", "mathematics", "science"] + +[lib] +crate-type = ["lib", "staticlib"] +bench = false + +[dependencies] + +[dev-dependencies] diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md new file mode 100644 index 0000000..0db52d9 --- /dev/null +++ b/crates/consistent-hashing/README.md @@ -0,0 +1,60 @@ +# Consistent Hashing + +Consistent hashing maps keys to a changing set of nodes (shards, servers) so that when nodes join or leave, only a small fraction of keys move. It is used in distributed caches, databases, object stores, and load balancers to achieve scalability and high availability with minimal data reshuffling. + +Common algorithms +- [Consistent hashing](https://en.wikipedia.org/wiki/Consistent_hashing) (hash ring with virtual nodes) +- [Rendezvous hashing](https://en.wikipedia.org/wiki/Rendezvous_hashing) +- [Jump consistent hash](https://en.wikipedia.org/wiki/Jump_consistent_hash) +- [Maglev hashing](https://research.google/pubs/pub44824) +- [AnchorHash: A Scalable Consistent Hash](https://arxiv.org/abs/1812.09674) +- [DXHash](https://arxiv.org/abs/2107.07930) +- [JumpBackHash](https://arxiv.org/abs/2403.18682) + +## Complexity summary + +where `N` is the number of nodes and `R` is the number of replicas. + +| Algorithm | Lookup per key | Node add/remove | Memory | Replication support | +|-------------------------|----------------------|----------------------------------------|---------------------------|--------------------------------------------------| +| Hash ring (with vnodes) | O(log N) binary search over N points; O(1) with specialized structures | O(log N) to insert/remove points | O(N) points | Yes: take next R distinct successors; O(log N + R) | +| Rendezvous | O(N) score per node; top-1 | O(1) (no state to rebalance) | O(N) node list | Yes: pick top R scores; O(N log R) | +| Jump consistent hash | O(log(N)) | O(1) | O(1) | Not native | +| AnchorHash | O(1) expected | O(1) expected/amortized | O(N) | Not native | +| DXHash | O(1) expected | O(1) expected | O(N) | Not native | +| JumpBackHash | O(1) | O(1) expected | O(1) | Not native | + +Replication of keys +- Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas evenly and avoid hotspots. +- Rendezvous hashing: replicate by selecting the top R nodes by score for the key. This naturally yields R distinct owners and supports weights. +- Jump consistent hash: the base function returns one bucket. Replication can be achieved by hashing (key, replica_index) and collecting R distinct buckets; this is simple but lacks the single-pass global ranking HRW provides. + +Why replication matters +- Tolerates node failures and maintenance without data unavailability. +- Distributes read/write load across multiple owners, reducing hotspots. +- Enables fast recovery and higher tail-latency resilience. + +## N-Choose-R replication + +We define the consistent `n-choose-rk` replication as follows: + +1. for a given number `n` of nodes, choose `k` distinct nodes `S`. +2. for a given `key` the chosen set of nodes must be uniformly chosen from all possible sets of size `k`. +3. when `n` increases by one, exactly one node in the chosen set will be changed with probability `k/(n+1)`. + +For simplicity, nodes are represented by integers `0..n`. +Given `k` independent consistent hash functions `h_i(n)` for a given key, the following algorithm will have the desired properties: + +``` +fn consistent_choose_k(key: Key, k: usize, n: usize) -> Vec { + (0..k).rev().scan(n, |n, k| Some(consistent_choose_next(key, k, n))).collect() +} + +fn consistent_choose_next(key: Key, k: usize, n: usize) -> usize { + (0..k).map(|k| consistent_hash(key, k, n - k) + k).max() +} + +fn consistent_hash(key: Key, k: usize, n: usize) -> usize { + // compute the k-th independent consistent hash for `key` and `n` nodes. +} +``` diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs new file mode 100644 index 0000000..c2f9e7a --- /dev/null +++ b/crates/consistent-hashing/src/lib.rs @@ -0,0 +1,292 @@ +use std::hash::{DefaultHasher, Hash, Hasher}; + +/// One building block for the consistent hashing algorithm is a consistent +/// hash iterator which enumerates all the hashes for a given for a specific bucket. +/// A bucket covers the range `(1< Self { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + bit.hash(&mut hasher); + Self { + hasher, + n, + is_first: true, + bit, + } + } +} + +impl Iterator for BucketIterator { + type Item = usize; + + fn next(&mut self) -> Option { + if self.bit == 0 { + return None; + } + if self.is_first { + let res = self.hasher.finish() % self.bit + self.bit; + if res < self.n as u64 { + self.n = res as usize; + return Some(self.n); + } + self.is_first = false; + } + loop { + 478392.hash(&mut self.hasher); + let res = self.hasher.finish() % (self.bit * 2); + if res & self.bit == 0 { + return None; + } + if res < self.n as u64 { + self.n = res as usize; + return Some(self.n); + } + } + } +} + +/// An iterator which enumerates all the consistent hashes for a given key +/// from largest to smallest in the range `0..n`. +pub struct ConsistentHashRevIterator { + bits: u64, + key: u64, + n: usize, + inner: BucketIterator, +} + +impl ConsistentHashRevIterator { + pub fn new(key: u64, n: usize) -> Self { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + let bits = hasher.finish() % n.next_power_of_two() as u64; + let inner = BucketIterator::default(); + Self { + bits, + key, + n, + inner, + } + } +} + +impl Iterator for ConsistentHashRevIterator { + type Item = usize; + + fn next(&mut self) -> Option { + if self.n == 0 { + return None; + } + if let Some(res) = self.inner.next() { + return Some(res); + } + while self.bits > 0 { + let bit = 1 << self.bits.ilog2(); + self.bits ^= bit; + self.inner = BucketIterator::new(self.key, self.n, bit); + if let Some(res) = self.inner.next() { + return Some(res); + } + } + self.n = 0; + Some(self.n) + } +} + +/// Same as `ConsistentHashRevIterator`, but iterates from smallest to largest +/// for the range `n..`. +pub struct ConsistentHashIterator { + bits: u64, + key: u64, + n: usize, + stack: Vec, +} + +impl ConsistentHashIterator { + pub fn new(key: u64, n: usize) -> Self { + let mut hasher = DefaultHasher::new(); + key.hash(&mut hasher); + let mut bits = hasher.finish() as u64; + bits &= !((n + 2).next_power_of_two() as u64 / 2 - 1); + let stack = if n == 0 { vec![0] } else { vec![] }; + Self { + bits, + key, + n, + stack, + } + } +} + +impl Iterator for ConsistentHashIterator { + type Item = usize; + + fn next(&mut self) -> Option { + if let Some(res) = self.stack.pop() { + return Some(res); + } + while self.bits > 0 { + let bit = self.bits & !(self.bits - 1); + self.bits &= self.bits - 1; + let inner = BucketIterator::new(self.key, bit as usize * 2, bit); + self.stack = inner.take_while(|x| *x >= self.n).collect(); + if let Some(res) = self.stack.pop() { + return Some(res); + } + } + None + } +} + +/// Wrapper around `ConsistentHashIterator` and `ConsistentHashRevIterator` to compute +/// the next or previous consistent hash for a given key for a given number of nodes `n`. +pub struct ConsistentHasher { + key: u64, +} + +impl ConsistentHasher { + pub fn new(key: u64) -> Self { + Self { key } + } + + pub fn prev(&self, n: usize) -> usize { + let mut sampler = ConsistentHashRevIterator::new(self.key, n); + sampler.next().expect("n must be > 0!") + } + + pub fn next(&self, n: usize) -> usize { + let mut sampler = ConsistentHashIterator::new(self.key, n); + sampler.next().expect("Exceeded iterator bounds :(") + } +} + +/// Implementation of a consistent choose k hashing algorithm. +/// It returns k distinct consistent hashes in the range `0..n`. +/// The hashes are consistent when `n` changes and when `k` changes! +/// I.e. on average exactly `1/(n+1)` (resp. `1/(k+1)`) many hashes will change +/// when `n` (resp. `k`) increases by one. Additionally, the returned `k` tuple +/// is guaranteed to be uniformely chosen from all possible `n-choose-k` tuples. +pub struct ConsistentChooseKHasher { + key: u64, + k: usize, +} + +impl ConsistentChooseKHasher { + pub fn new(key: u64, k: usize) -> Self { + Self { key, k } + } + + // TODO: Implement this as an iterator! + pub fn prev(&self, mut n: usize) -> Vec { + let mut samples = Vec::with_capacity(self.k); + let mut samplers: Vec<_> = (0..self.k) + .map(|i| ConsistentHashRevIterator::new(self.key + 43987492 * i as u64, n - i).peekable()) + .collect(); + for i in (0..self.k).rev() { + let mut max = 0; + for k in 0..=i { + while samplers[k].peek() >= Some(&(n - k)) && n - k > 0 { + samplers[k].next(); + } + max = max.max(samplers[k].peek().unwrap() + k); + } + samples.push(max); + n = max; + } + samples.sort(); + samples + } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_uniform_1() { + for k in 0..100 { + let sampler = ConsistentHasher::new(k); + for n in 0..1000 { + assert!(sampler.prev(n + 1) <= sampler.prev(n + 2)); + let next = sampler.next(n); + assert_eq!(next, sampler.prev(next + 1)); + } + let mut iter_rev: Vec<_> = ConsistentHashIterator::new(k, 0) + .take_while(|x| *x < 1000) + .collect(); + iter_rev.reverse(); + let iter: Vec<_> = ConsistentHashRevIterator::new(k, 1000).collect(); + assert_eq!(iter, iter_rev); + } + let mut stats = vec![0; 13]; + for i in 0..100000 { + let sampler = ConsistentHasher::new(i); + let x = sampler.prev(stats.len()); + stats[x] += 1; + } + println!("{stats:?}"); + } + + #[test] + fn test_uniform_k() { + const K: usize = 3; + for k in 0..100 { + let sampler = ConsistentChooseKHasher::new(k, K); + for n in K..1000 { + let samples = sampler.prev(n + 1); + assert!(samples.len() == K); + for i in 0..K - 1 { + assert!(samples[i] < samples[i + 1]); + } + let next = sampler.prev(n + 2); + for i in 0..K { + assert!(samples[i] <= next[i]); + } + let mut merged = samples.clone(); + merged.extend(next.clone()); + merged.sort(); + merged.dedup(); + assert!( + merged.len() == K || merged.len() == K + 1, + "Unexpected {samples:?} vs. {next:?}" + ); + } + } + let mut stats = vec![0; 8]; + for i in 0..32 { + let sampler = ConsistentChooseKHasher::new(i + 32783, 2); + let samples = sampler.prev(stats.len()); + for s in samples { + stats[s] += 1; + } + } + println!("{stats:?}"); + // Test consistency when increasing k! + for k in 1..10 { + for n in k + 1..20 { + for key in 0..1000 { + let sampler1 = ConsistentChooseKHasher::new(key, k); + let sampler2 = ConsistentChooseKHasher::new(key, k + 1); + let set1 = sampler1.prev(n); + let set2 = sampler2.prev(n); + assert_eq!(set1.len(), k); + assert_eq!(set2.len(), k + 1); + let mut merged = set1.clone(); + merged.extend(set2); + merged.sort(); + merged.dedup(); + assert_eq!(merged.len(), k + 1); + } + } + } + } +} From 89f8ad42a11a3b3a7ffa3aa5d2cd9b5e093ed9c7 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Tue, 12 Aug 2025 09:42:37 +0200 Subject: [PATCH 02/20] Update README.md --- crates/consistent-hashing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index 0db52d9..27b6d00 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -43,7 +43,7 @@ We define the consistent `n-choose-rk` replication as follows: 3. when `n` increases by one, exactly one node in the chosen set will be changed with probability `k/(n+1)`. For simplicity, nodes are represented by integers `0..n`. -Given `k` independent consistent hash functions `h_i(n)` for a given key, the following algorithm will have the desired properties: +Given `k` independent consistent hash functions `consistent_hash(key, k, n)` for a given `key`, the following algorithm will have the desired properties: ``` fn consistent_choose_k(key: Key, k: usize, n: usize) -> Vec { From b03f0b70ffe37912c2ca23bff5f56b57e42d9c94 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Tue, 12 Aug 2025 10:23:09 +0200 Subject: [PATCH 03/20] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- crates/consistent-hashing/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs index c2f9e7a..d3bf878 100644 --- a/crates/consistent-hashing/src/lib.rs +++ b/crates/consistent-hashing/src/lib.rs @@ -1,7 +1,7 @@ use std::hash::{DefaultHasher, Hash, Hasher}; /// One building block for the consistent hashing algorithm is a consistent -/// hash iterator which enumerates all the hashes for a given for a specific bucket. +/// hash iterator which enumerates all the hashes for a specific bucket. /// A bucket covers the range `(1< Date: Wed, 13 Aug 2025 10:47:04 +0200 Subject: [PATCH 04/20] finish proof --- crates/consistent-hashing/README.md | 36 +++++++++++++++++++++------- crates/consistent-hashing/src/lib.rs | 14 +++++------ 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index 27b6d00..ac238a0 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -36,25 +36,45 @@ Why replication matters ## N-Choose-R replication -We define the consistent `n-choose-rk` replication as follows: +We define the consistent `n-choose-k` replication as follows: -1. for a given number `n` of nodes, choose `k` distinct nodes `S`. -2. for a given `key` the chosen set of nodes must be uniformly chosen from all possible sets of size `k`. -3. when `n` increases by one, exactly one node in the chosen set will be changed with probability `k/(n+1)`. +1. For a given number `n` of nodes, choose `k` distinct nodes `S`. +2. For a given `key` the chosen set of nodes must be uniformly chosen from all possible sets of size `k`. +3. When `n` increases by one, exactly one node in the chosen set will be changed. +4. and the node will be changed with probability `k/(n+1)`. For simplicity, nodes are represented by integers `0..n`. Given `k` independent consistent hash functions `consistent_hash(key, k, n)` for a given `key`, the following algorithm will have the desired properties: ``` fn consistent_choose_k(key: Key, k: usize, n: usize) -> Vec { - (0..k).rev().scan(n, |n, k| Some(consistent_choose_next(key, k, n))).collect() + (0..k).rev().scan(n, |n, k| Some(consistent_choose_max(key, k + 1, n))).collect() } -fn consistent_choose_next(key: Key, k: usize, n: usize) -> usize { +fn consistent_choose_max(key: Key, k: usize, n: usize) -> usize { (0..k).map(|k| consistent_hash(key, k, n - k) + k).max() } -fn consistent_hash(key: Key, k: usize, n: usize) -> usize { - // compute the k-th independent consistent hash for `key` and `n` nodes. +fn consistent_hash(key: Key, i: usize, n: usize) -> usize { + // compute the i-th independent consistent hash for `key` and `n` nodes. } ``` + +Let's define `M(k,n) = consistent_choose_max(_, k, n)` and `S(k, n) := consistent_choose_k(_, k, n)` as short-cuts for some arbitrary fixed `key`. + +Since `M(k, n) < n` and `S(k, n) = {M(k, n)} ∪ S(k - 1, M(k, n))` for `k > 1`, `S(k, n)` constructs a strictly monotonically decreasing sequence. The sequence outputs exactly `k` elements which therefore must all be distinct which proves property 1 for `k <= n`. + +Properties 2, 3, and 4 can be proven via induction as follows. + +`k = 1`: We expect that `consistent_hash` returns a single uniformly distributed node index which is consistent in `n`, i.e. changes the hash value with probability `1/(n+1)`, when `n` increments by one. In our implementation, we use an `O(1)` implementation of the jump-hash algorithm. For `k=1`, `consistent_choose_k(key, 1, n)` becomes a single function call to `consistent_choose_max(key, 1, n)` which in turn calls `consistent_hash(key, 0, n)`. I.e. `consistent_choose_k` inherits the all the desired properties from `consistent_hash` for `k=1` and all `n>=1`. + +`k -> k+1`: `M(k+1, n+1) = M(k+1, n)` iff `M(k, n+1) < n` and `consistent_hash(_, k, n+1-k) < n - k`. The probability for this is `(n+1-k)/(n+1)` for the former by induction and `(n-k)/(n+1-k)` by the assumption that `consistent_hash` is a proper consistent hash function. Since both these probabilities are assumed to be independent, the probability that our initial value changes is `1 - (n+1-k)/(n+1) * (n-k)/(n+1-k) = 1 - (n-k)/(n+1) = (k+1)/(n+1)` proving property 4. + +Property 3 is trivially satisfied if `S(k+1, n+1) = S(k+1, n)`. So, we focus on the case where `S(k+1, n+1) != S(k+1, n)`, which implies that `n ∈ S(k+1, n+1)` as largest element. +We know that `S(k+1, n) = {m} ∪ S(k, m)` for some `m` by definition and `S(k, n) = S(k, u) ∖ {v} ∪ {w}` by induction for some `u`, `v`, and `w`. Thus far we have `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, u) ∖ {v} ∪ {w}`. + +If `u == m`, then `S(k+1, n) = {m} ∪ S(k, m) ∖ {v} ∪ {w}` and `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, m) ∖ {v} ∪ {w}` and the two differ exaclty in the elemetns `m` and `n` proving property 3. + +If `u != m`, then `consistent_hash(_, k, n) = m`, since that's the only way how the largest values in `S(k+1, n)` and `S(k, n)` can differ. In this case, `m ∉ S(k+1, n+1)`, since `n` (and not `m`) is the largest element of `S(k+1, n+1)`. Furthermore, `S(k, n) = S(k, m)`, since `consistent_hash(_, i, n) < m` for all `i < k` (otherwise there is a contradiction). +Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m} ∪ S(k, m)` which differ exactly in the elements `n` and `m` which concludes the proof. + diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs index c2f9e7a..8ae98ec 100644 --- a/crates/consistent-hashing/src/lib.rs +++ b/crates/consistent-hashing/src/lib.rs @@ -157,14 +157,14 @@ impl ConsistentHasher { Self { key } } - pub fn prev(&self, n: usize) -> usize { + pub fn prev(&self, n: usize) -> Option { let mut sampler = ConsistentHashRevIterator::new(self.key, n); - sampler.next().expect("n must be > 0!") + sampler.next() } - pub fn next(&self, n: usize) -> usize { + pub fn next(&self, n: usize) -> Option { let mut sampler = ConsistentHashIterator::new(self.key, n); - sampler.next().expect("Exceeded iterator bounds :(") + sampler.next() } } @@ -217,8 +217,8 @@ mod tests { let sampler = ConsistentHasher::new(k); for n in 0..1000 { assert!(sampler.prev(n + 1) <= sampler.prev(n + 2)); - let next = sampler.next(n); - assert_eq!(next, sampler.prev(next + 1)); + let next = sampler.next(n).unwrap(); + assert_eq!(next, sampler.prev(next + 1).unwrap()); } let mut iter_rev: Vec<_> = ConsistentHashIterator::new(k, 0) .take_while(|x| *x < 1000) @@ -230,7 +230,7 @@ mod tests { let mut stats = vec![0; 13]; for i in 0..100000 { let sampler = ConsistentHasher::new(i); - let x = sampler.prev(stats.len()); + let x = sampler.prev(stats.len()).unwrap(); stats[x] += 1; } println!("{stats:?}"); From a5eb91eeb68300a15a945a90d1499a7ffbe1a7e6 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 13 Aug 2025 11:24:22 +0200 Subject: [PATCH 05/20] Update README.md --- crates/consistent-hashing/README.md | 58 +++++++++++++++++++---------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index ac238a0..b0bacf3 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -15,36 +15,33 @@ Common algorithms where `N` is the number of nodes and `R` is the number of replicas. -| Algorithm | Lookup per key | Node add/remove | Memory | Replication support | -|-------------------------|----------------------|----------------------------------------|---------------------------|--------------------------------------------------| -| Hash ring (with vnodes) | O(log N) binary search over N points; O(1) with specialized structures | O(log N) to insert/remove points | O(N) points | Yes: take next R distinct successors; O(log N + R) | -| Rendezvous | O(N) score per node; top-1 | O(1) (no state to rebalance) | O(N) node list | Yes: pick top R scores; O(N log R) | -| Jump consistent hash | O(log(N)) | O(1) | O(1) | Not native | -| AnchorHash | O(1) expected | O(1) expected/amortized | O(N) | Not native | -| DXHash | O(1) expected | O(1) expected | O(N) | Not native | -| JumpBackHash | O(1) | O(1) expected | O(1) | Not native | +| Algorithm | Lookup per key | Node add/remove | Memory | Lookup with replication | +| | (no replication) | | | | +|-------------------------|---------------------|----------------------------------------|---------------------------|-------------------------------------| +| Hash ring (with vnodes) | O(log N): binary search over N points; O(1): with specialized structures | O(log N) | O(N) | O(log N + R): Take next R distinct successors | +| Rendezvous | O(N): max score | O(1) | O(N) node list | O(N log R): pick top R scores | +| Jump consistent hash | O(log(N)) expected | 0 | O(1) | Not native | +| AnchorHash | O(1) expected | O(1)? | O(N)? | Not native | +| DXHash | O(1) expected | O(1)? | O(N)? | Not native | +| JumpBackHash | O(1) expected | 0 | O(1) | Not native | +| $ConsistentChooseK$ | $O(1) expected$ | $0$ | $O(1)$ | $O(R^2)$; $O(R log(R))$: using heap | Replication of keys -- Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas evenly and avoid hotspots. +- Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas more evenly. Replicas are not independently distributed. - Rendezvous hashing: replicate by selecting the top R nodes by score for the key. This naturally yields R distinct owners and supports weights. -- Jump consistent hash: the base function returns one bucket. Replication can be achieved by hashing (key, replica_index) and collecting R distinct buckets; this is simple but lacks the single-pass global ranking HRW provides. +- Jump consistent hash and variatns: the base function returns one bucket. Replication can be achieved by hashing (key, replica_index) and collecting R distinct buckets; this is simple but loses the consistency property! +- ConsistentChooseK: Faster and more memory efficient than all other solutions. Why replication matters - Tolerates node failures and maintenance without data unavailability. - Distributes read/write load across multiple owners, reducing hotspots. - Enables fast recovery and higher tail-latency resilience. -## N-Choose-R replication - -We define the consistent `n-choose-k` replication as follows: - -1. For a given number `n` of nodes, choose `k` distinct nodes `S`. -2. For a given `key` the chosen set of nodes must be uniformly chosen from all possible sets of size `k`. -3. When `n` increases by one, exactly one node in the chosen set will be changed. -4. and the node will be changed with probability `k/(n+1)`. +## ConsistentChooseK algorithm -For simplicity, nodes are represented by integers `0..n`. -Given `k` independent consistent hash functions `consistent_hash(key, k, n)` for a given `key`, the following algorithm will have the desired properties: +The following functions summarize the core algorithmic innovation as a minimal Rust excerpt. +`n` is the number of nodes and `k` is the number of desired replica. +The chosen nodes are returned as distinct integers in the range `0..n`. ``` fn consistent_choose_k(key: Key, k: usize, n: usize) -> Vec { @@ -60,7 +57,28 @@ fn consistent_hash(key: Key, i: usize, n: usize) -> usize { } ``` +`consistent_choose_k` makes `k` calls to `consistent_choose_max` which calls `consistent_hash` another `k` times. +In total, `consistent_hash` is called `k * (k+1) / 2` Utilizing a `O(1)` solution for `consistent_hash` leads to a `O(k^2)` runtime. +This runtime can be further improved by replacing the max operation with a heap where popped elements are updated according to the new arguments `n` and `k`. +With this optimization, the complexity reduces to `O(k log k)`. +With some probabilistic bucketing strategy, it should be possible to reduce the expected runtime to `O(k)`. +For small `k` neither optimization is probably improving the actual performance though. + +The next section proves why this simple code works. + +## N-Choose-R replication + +We define the consistent `n-choose-k` replication as follows: + +1. For a given number `n` of nodes, choose `k` distinct nodes `S`. +2. For a given `key` the chosen set of nodes must be uniformly chosen from all possible sets of size `k`. +3. When `n` increases by one, exactly one node in the chosen set will be changed. +4. and the node will be changed with probability `k/(n+1)`. + +In the remainder of this section we prove that the `consistent_choose_k` algorithm satisfies those properties. + Let's define `M(k,n) = consistent_choose_max(_, k, n)` and `S(k, n) := consistent_choose_k(_, k, n)` as short-cuts for some arbitrary fixed `key`. +We assume that `consistent_hash(key, k, n)` computes `k` independent consistent hash functions. Since `M(k, n) < n` and `S(k, n) = {M(k, n)} ∪ S(k - 1, M(k, n))` for `k > 1`, `S(k, n)` constructs a strictly monotonically decreasing sequence. The sequence outputs exactly `k` elements which therefore must all be distinct which proves property 1 for `k <= n`. From 220624d716d15e06ba451eaa9f7178e08537fef6 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 13 Aug 2025 11:26:35 +0200 Subject: [PATCH 06/20] Update README.md --- crates/consistent-hashing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index b0bacf3..c9b7484 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -24,7 +24,7 @@ where `N` is the number of nodes and `R` is the number of replicas. | AnchorHash | O(1) expected | O(1)? | O(N)? | Not native | | DXHash | O(1) expected | O(1)? | O(N)? | Not native | | JumpBackHash | O(1) expected | 0 | O(1) | Not native | -| $ConsistentChooseK$ | $O(1) expected$ | $0$ | $O(1)$ | $O(R^2)$; $O(R log(R))$: using heap | +| $$ConsistentChooseK$$ | $$O(1) expected$$ | $$0$$ | $$O(1)$$ | $$O(R^2)$$; $$O(R log(R))$$: using heap | Replication of keys - Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas more evenly. Replicas are not independently distributed. From fc69a9eaab94cb9f5cb20fb9bb63dfec73857498 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 13 Aug 2025 11:33:25 +0200 Subject: [PATCH 07/20] Update README.md --- crates/consistent-hashing/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index c9b7484..27e5c2f 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -5,7 +5,7 @@ Consistent hashing maps keys to a changing set of nodes (shards, servers) so tha Common algorithms - [Consistent hashing](https://en.wikipedia.org/wiki/Consistent_hashing) (hash ring with virtual nodes) - [Rendezvous hashing](https://en.wikipedia.org/wiki/Rendezvous_hashing) -- [Jump consistent hash](https://en.wikipedia.org/wiki/Jump_consistent_hash) +- [Jump consistent hash](https://arxiv.org/pdf/1406.2294) - [Maglev hashing](https://research.google/pubs/pub44824) - [AnchorHash: A Scalable Consistent Hash](https://arxiv.org/abs/1812.09674) - [DXHash](https://arxiv.org/abs/2107.07930) @@ -24,7 +24,7 @@ where `N` is the number of nodes and `R` is the number of replicas. | AnchorHash | O(1) expected | O(1)? | O(N)? | Not native | | DXHash | O(1) expected | O(1)? | O(N)? | Not native | | JumpBackHash | O(1) expected | 0 | O(1) | Not native | -| $$ConsistentChooseK$$ | $$O(1) expected$$ | $$0$$ | $$O(1)$$ | $$O(R^2)$$; $$O(R log(R))$$: using heap | +| **ConsistentChooseK** | **O(1) expected** | **0** | **O(1)** | **O(R^2)**; **O(R log(R))**: using heap | Replication of keys - Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas more evenly. Replicas are not independently distributed. @@ -64,7 +64,7 @@ With this optimization, the complexity reduces to `O(k log k)`. With some probabilistic bucketing strategy, it should be possible to reduce the expected runtime to `O(k)`. For small `k` neither optimization is probably improving the actual performance though. -The next section proves why this simple code works. +The next section proves the correctness of this algorithm. ## N-Choose-R replication @@ -91,8 +91,8 @@ Properties 2, 3, and 4 can be proven via induction as follows. Property 3 is trivially satisfied if `S(k+1, n+1) = S(k+1, n)`. So, we focus on the case where `S(k+1, n+1) != S(k+1, n)`, which implies that `n ∈ S(k+1, n+1)` as largest element. We know that `S(k+1, n) = {m} ∪ S(k, m)` for some `m` by definition and `S(k, n) = S(k, u) ∖ {v} ∪ {w}` by induction for some `u`, `v`, and `w`. Thus far we have `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, u) ∖ {v} ∪ {w}`. -If `u == m`, then `S(k+1, n) = {m} ∪ S(k, m) ∖ {v} ∪ {w}` and `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, m) ∖ {v} ∪ {w}` and the two differ exaclty in the elemetns `m` and `n` proving property 3. +If `u = m`, then `S(k+1, n) = {m} ∪ S(k, m) ∖ {v} ∪ {w}` and `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, m) ∖ {v} ∪ {w}` and the two differ exaclty in the elemetns `m` and `n` proving property 3. -If `u != m`, then `consistent_hash(_, k, n) = m`, since that's the only way how the largest values in `S(k+1, n)` and `S(k, n)` can differ. In this case, `m ∉ S(k+1, n+1)`, since `n` (and not `m`) is the largest element of `S(k+1, n+1)`. Furthermore, `S(k, n) = S(k, m)`, since `consistent_hash(_, i, n) < m` for all `i < k` (otherwise there is a contradiction). +If `u ≠ m`, then `consistent_hash(_, k, n) = m`, since that's the only way how the largest values in `S(k+1, n)` and `S(k, n)` can differ. In this case, `m ∉ S(k+1, n+1)`, since `n` (and not `m`) is the largest element of `S(k+1, n+1)`. Furthermore, `S(k, n) = S(k, m)`, since `consistent_hash(_, i, n) < m` for all `i < k` (otherwise there is a contradiction). Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m} ∪ S(k, m)` which differ exactly in the elements `n` and `m` which concludes the proof. From 90259e92e127907a670cb36bcaafe541cf7e42cf Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 13 Aug 2025 11:45:18 +0200 Subject: [PATCH 08/20] Update README.md --- crates/consistent-hashing/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index 27e5c2f..bcd137a 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -20,16 +20,17 @@ where `N` is the number of nodes and `R` is the number of replicas. |-------------------------|---------------------|----------------------------------------|---------------------------|-------------------------------------| | Hash ring (with vnodes) | O(log N): binary search over N points; O(1): with specialized structures | O(log N) | O(N) | O(log N + R): Take next R distinct successors | | Rendezvous | O(N): max score | O(1) | O(N) node list | O(N log R): pick top R scores | -| Jump consistent hash | O(log(N)) expected | 0 | O(1) | Not native | -| AnchorHash | O(1) expected | O(1)? | O(N)? | Not native | -| DXHash | O(1) expected | O(1)? | O(N)? | Not native | +| Jump consistent hash | O(log(N)) expected | 0 | O(1) | O(R log N) | +| AnchorHash | O(1) expected | O(1) | O(N) | Not native | +| DXHash | O(1) expected | O(1) | O(N) | Not native | | JumpBackHash | O(1) expected | 0 | O(1) | Not native | | **ConsistentChooseK** | **O(1) expected** | **0** | **O(1)** | **O(R^2)**; **O(R log(R))**: using heap | Replication of keys - Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas more evenly. Replicas are not independently distributed. - Rendezvous hashing: replicate by selecting the top R nodes by score for the key. This naturally yields R distinct owners and supports weights. -- Jump consistent hash and variatns: the base function returns one bucket. Replication can be achieved by hashing (key, replica_index) and collecting R distinct buckets; this is simple but loses the consistency property! +- Jump consistent hash: the base function doesn't support replication. But the math can be easily modified to support consistent replication. +- JumpBackHash and variants: The trick of Jump consistent hash to support replication won't work here due to the additional state introduced. - ConsistentChooseK: Faster and more memory efficient than all other solutions. Why replication matters From 8480ea3ed645c6944be006cf91fb6d8c50530cc0 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 13 Aug 2025 15:55:20 +0200 Subject: [PATCH 09/20] Replace key with hasher traits --- crates/consistent-hashing/README.md | 7 +- crates/consistent-hashing/src/lib.rs | 190 +++++++++++++++++---------- 2 files changed, 127 insertions(+), 70 deletions(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index bcd137a..4f29c56 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -67,7 +67,7 @@ For small `k` neither optimization is probably improving the actual performance The next section proves the correctness of this algorithm. -## N-Choose-R replication +## N-Choose-K replication We define the consistent `n-choose-k` replication as follows: @@ -87,7 +87,7 @@ Properties 2, 3, and 4 can be proven via induction as follows. `k = 1`: We expect that `consistent_hash` returns a single uniformly distributed node index which is consistent in `n`, i.e. changes the hash value with probability `1/(n+1)`, when `n` increments by one. In our implementation, we use an `O(1)` implementation of the jump-hash algorithm. For `k=1`, `consistent_choose_k(key, 1, n)` becomes a single function call to `consistent_choose_max(key, 1, n)` which in turn calls `consistent_hash(key, 0, n)`. I.e. `consistent_choose_k` inherits the all the desired properties from `consistent_hash` for `k=1` and all `n>=1`. -`k -> k+1`: `M(k+1, n+1) = M(k+1, n)` iff `M(k, n+1) < n` and `consistent_hash(_, k, n+1-k) < n - k`. The probability for this is `(n+1-k)/(n+1)` for the former by induction and `(n-k)/(n+1-k)` by the assumption that `consistent_hash` is a proper consistent hash function. Since both these probabilities are assumed to be independent, the probability that our initial value changes is `1 - (n+1-k)/(n+1) * (n-k)/(n+1-k) = 1 - (n-k)/(n+1) = (k+1)/(n+1)` proving property 4. +`k → k+1`: `M(k+1, n+1) = M(k+1, n)` iff `M(k, n+1) < n` and `consistent_hash(_, k, n+1-k) < n - k`. The probability for this is `(n+1-k)/(n+1)` for the former by induction and `(n-k)/(n+1-k)` by the assumption that `consistent_hash` is a proper consistent hash function. Since both these probabilities are assumed to be independent, the probability that our initial value changes is `1 - (n+1-k)/(n+1) * (n-k)/(n+1-k) = 1 - (n-k)/(n+1) = (k+1)/(n+1)` proving property 4. Property 3 is trivially satisfied if `S(k+1, n+1) = S(k+1, n)`. So, we focus on the case where `S(k+1, n+1) != S(k+1, n)`, which implies that `n ∈ S(k+1, n+1)` as largest element. We know that `S(k+1, n) = {m} ∪ S(k, m)` for some `m` by definition and `S(k, n) = S(k, u) ∖ {v} ∪ {w}` by induction for some `u`, `v`, and `w`. Thus far we have `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, u) ∖ {v} ∪ {w}`. @@ -95,5 +95,4 @@ We know that `S(k+1, n) = {m} ∪ S(k, m)` for some `m` by definition and `S(k, If `u = m`, then `S(k+1, n) = {m} ∪ S(k, m) ∖ {v} ∪ {w}` and `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, m) ∖ {v} ∪ {w}` and the two differ exaclty in the elemetns `m` and `n` proving property 3. If `u ≠ m`, then `consistent_hash(_, k, n) = m`, since that's the only way how the largest values in `S(k+1, n)` and `S(k, n)` can differ. In this case, `m ∉ S(k+1, n+1)`, since `n` (and not `m`) is the largest element of `S(k+1, n+1)`. Furthermore, `S(k, n) = S(k, m)`, since `consistent_hash(_, i, n) < m` for all `i < k` (otherwise there is a contradiction). -Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m} ∪ S(k, m)` which differ exactly in the elements `n` and `m` which concludes the proof. - +Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m} ∪ S(k, m)` which differ exactly in the elements `n` and `m` which concludes the proof. \ No newline at end of file diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs index 647215c..dd7675c 100644 --- a/crates/consistent-hashing/src/lib.rs +++ b/crates/consistent-hashing/src/lib.rs @@ -1,21 +1,76 @@ -use std::hash::{DefaultHasher, Hash, Hasher}; +use std::hash::{Hash, Hasher}; + +/// A trait which behaves like a pseudo-random number generator. +/// It is used to generate consistent hashes within one bucket. +/// Note: the hasher must have been seeded with the key during construction. +pub trait HashSequence { + fn next(&mut self) -> u64; +} + +/// A trait for building a special bit mask and sequences of hashes for different bit positions. +/// Note: the hasher must have been seeded with the key during construction. +pub trait HashSeqBuilder { + type Seq: HashSequence; + + fn bit_mask(&self) -> u64; + /// Return a HashSequence instance which is seeded with the given bit position + /// and the seed of this builder. + fn hash_seq(&self, bit: u64) -> Self::Seq; +} + +/// A trait for building multiple independent hash builders +/// Note: the hasher must have been seeded with the key during construction. +pub trait ManySeqBuilder { + type Builder: HashSeqBuilder; + + /// Returns the i-th independent hash builder. + fn seq_builder(&self, i: usize) -> Self::Builder; +} + +impl HashSequence for H { + fn next(&mut self) -> u64 { + 54387634019u64.hash(self); + self.finish() + } +} + +impl HashSeqBuilder for H { + type Seq = H; + + fn bit_mask(&self) -> u64 { + self.finish() + } + + fn hash_seq(&self, bit: u64) -> Self::Seq { + let mut hasher = self.clone(); + bit.hash(&mut hasher); + hasher + } +} + +impl ManySeqBuilder for H { + type Builder = H; + + fn seq_builder(&self, i: usize) -> Self::Builder { + let mut hasher = self.clone(); + i.hash(&mut hasher); + hasher + } +} /// One building block for the consistent hashing algorithm is a consistent /// hash iterator which enumerates all the hashes for a specific bucket. /// A bucket covers the range `(1< { + hasher: H, n: usize, is_first: bool, - bit: u64, + bit: u64, // A bitmask with a single bit set. } -impl BucketIterator { - fn new(key: u64, n: usize, bit: u64) -> Self { - let mut hasher = DefaultHasher::new(); - key.hash(&mut hasher); - bit.hash(&mut hasher); +impl BucketIterator { + fn new(n: usize, bit: u64, hasher: H) -> Self { Self { hasher, n, @@ -25,7 +80,7 @@ impl BucketIterator { } } -impl Iterator for BucketIterator { +impl Iterator for BucketIterator { type Item = usize; fn next(&mut self) -> Option { @@ -33,16 +88,15 @@ impl Iterator for BucketIterator { return None; } if self.is_first { - let res = self.hasher.finish() % self.bit + self.bit; + let res = (self.hasher.next() & (self.bit - 1)) + self.bit; + self.is_first = false; if res < self.n as u64 { self.n = res as usize; return Some(self.n); } - self.is_first = false; } loop { - 478392.hash(&mut self.hasher); - let res = self.hasher.finish() % (self.bit * 2); + let res = self.hasher.next() & (self.bit * 2 - 1); if res & self.bit == 0 { return None; } @@ -56,77 +110,70 @@ impl Iterator for BucketIterator { /// An iterator which enumerates all the consistent hashes for a given key /// from largest to smallest in the range `0..n`. -pub struct ConsistentHashRevIterator { +pub struct ConsistentHashRevIterator { + builder: H, bits: u64, - key: u64, n: usize, - inner: BucketIterator, + inner: Option>, } -impl ConsistentHashRevIterator { - pub fn new(key: u64, n: usize) -> Self { - let mut hasher = DefaultHasher::new(); - key.hash(&mut hasher); - let bits = hasher.finish() % n.next_power_of_two() as u64; - let inner = BucketIterator::default(); +impl ConsistentHashRevIterator { + pub fn new(n: usize, builder: H) -> Self { Self { - bits, - key, + bits: builder.bit_mask() & (n.next_power_of_two() as u64 - 1), + builder, n, - inner, + inner: None, } } } -impl Iterator for ConsistentHashRevIterator { +impl Iterator for ConsistentHashRevIterator { type Item = usize; fn next(&mut self) -> Option { if self.n == 0 { return None; } - if let Some(res) = self.inner.next() { + if let Some(res) = self.inner.as_mut().and_then(|inner| inner.next()) { return Some(res); } while self.bits > 0 { let bit = 1 << self.bits.ilog2(); self.bits ^= bit; - self.inner = BucketIterator::new(self.key, self.n, bit); - if let Some(res) = self.inner.next() { + let seq = self.builder.hash_seq(bit); + let mut iter = BucketIterator::new(self.n, bit, seq); + if let Some(res) = iter.next() { + self.inner = Some(iter); return Some(res); } } self.n = 0; - Some(self.n) + Some(0) } } /// Same as `ConsistentHashRevIterator`, but iterates from smallest to largest /// for the range `n..`. -pub struct ConsistentHashIterator { +pub struct ConsistentHashIterator { bits: u64, - key: u64, n: usize, + builder: H, stack: Vec, } -impl ConsistentHashIterator { - pub fn new(key: u64, n: usize) -> Self { - let mut hasher = DefaultHasher::new(); - key.hash(&mut hasher); - let mut bits = hasher.finish() as u64; - bits &= !((n + 2).next_power_of_two() as u64 / 2 - 1); - let stack = if n == 0 { vec![0] } else { vec![] }; +impl ConsistentHashIterator { + pub fn new(n: usize, builder: H) -> Self { Self { - bits, - key, + bits: builder.bit_mask() & !((n + 2).next_power_of_two() as u64 / 2 - 1), + stack: if n == 0 { vec![0] } else { vec![] }, + builder, n, - stack, } } } -impl Iterator for ConsistentHashIterator { +impl Iterator for ConsistentHashIterator { type Item = usize; fn next(&mut self) -> Option { @@ -136,7 +183,7 @@ impl Iterator for ConsistentHashIterator { while self.bits > 0 { let bit = self.bits & !(self.bits - 1); self.bits &= self.bits - 1; - let inner = BucketIterator::new(self.key, bit as usize * 2, bit); + let inner = BucketIterator::new(bit as usize * 2, bit, self.builder.hash_seq(bit)); self.stack = inner.take_while(|x| *x >= self.n).collect(); if let Some(res) = self.stack.pop() { return Some(res); @@ -148,22 +195,22 @@ impl Iterator for ConsistentHashIterator { /// Wrapper around `ConsistentHashIterator` and `ConsistentHashRevIterator` to compute /// the next or previous consistent hash for a given key for a given number of nodes `n`. -pub struct ConsistentHasher { - key: u64, +pub struct ConsistentHasher { + builder: H, } -impl ConsistentHasher { - pub fn new(key: u64) -> Self { - Self { key } +impl ConsistentHasher { + pub fn new(builder: H) -> Self { + Self { builder } } pub fn prev(&self, n: usize) -> Option { - let mut sampler = ConsistentHashRevIterator::new(self.key, n); + let mut sampler = ConsistentHashRevIterator::new(n, self.builder.clone()); sampler.next() } pub fn next(&self, n: usize) -> Option { - let mut sampler = ConsistentHashIterator::new(self.key, n); + let mut sampler = ConsistentHashIterator::new(n, self.builder.clone()); sampler.next() } } @@ -174,21 +221,21 @@ impl ConsistentHasher { /// I.e. on average exactly `1/(n+1)` (resp. `1/(k+1)`) many hashes will change /// when `n` (resp. `k`) increases by one. Additionally, the returned `k` tuple /// is guaranteed to be uniformely chosen from all possible `n-choose-k` tuples. -pub struct ConsistentChooseKHasher { - key: u64, +pub struct ConsistentChooseKHasher { + builder: H, k: usize, } -impl ConsistentChooseKHasher { - pub fn new(key: u64, k: usize) -> Self { - Self { key, k } +impl ConsistentChooseKHasher { + pub fn new(builder: H, k: usize) -> Self { + Self { builder, k } } // TODO: Implement this as an iterator! pub fn prev(&self, mut n: usize) -> Vec { let mut samples = Vec::with_capacity(self.k); let mut samplers: Vec<_> = (0..self.k) - .map(|i| ConsistentHashRevIterator::new(self.key + 43987492 * i as u64, n - i).peekable()) + .map(|i| ConsistentHashRevIterator::new(n - i, self.builder.seq_builder(i)).peekable()) .collect(); for i in (0..self.k).rev() { let mut max = 0; @@ -211,25 +258,33 @@ impl ConsistentChooseKHasher { mod tests { use super::*; + fn hasher_for_key(key: u64) -> DefaultHasher { + let mut hasher = DefaultHasher::default(); + key.hash(&mut hasher); + hasher + } + #[test] fn test_uniform_1() { for k in 0..100 { - let sampler = ConsistentHasher::new(k); + let hasher = hasher_for_key(k); + let sampler = ConsistentHasher::new(hasher.clone()); for n in 0..1000 { assert!(sampler.prev(n + 1) <= sampler.prev(n + 2)); let next = sampler.next(n).unwrap(); assert_eq!(next, sampler.prev(next + 1).unwrap()); } - let mut iter_rev: Vec<_> = ConsistentHashIterator::new(k, 0) + let mut iter_rev: Vec<_> = ConsistentHashIterator::new(0, hasher.clone()) .take_while(|x| *x < 1000) .collect(); iter_rev.reverse(); - let iter: Vec<_> = ConsistentHashRevIterator::new(k, 1000).collect(); + let iter: Vec<_> = ConsistentHashRevIterator::new(1000, hasher).collect(); assert_eq!(iter, iter_rev); } let mut stats = vec![0; 13]; for i in 0..100000 { - let sampler = ConsistentHasher::new(i); + let hasher = hasher_for_key(i); + let sampler = ConsistentHasher::new(hasher); let x = sampler.prev(stats.len()).unwrap(); stats[x] += 1; } @@ -240,7 +295,8 @@ mod tests { fn test_uniform_k() { const K: usize = 3; for k in 0..100 { - let sampler = ConsistentChooseKHasher::new(k, K); + let hasher = hasher_for_key(k); + let sampler = ConsistentChooseKHasher::new(hasher, K); for n in K..1000 { let samples = sampler.prev(n + 1); assert!(samples.len() == K); @@ -263,7 +319,8 @@ mod tests { } let mut stats = vec![0; 8]; for i in 0..32 { - let sampler = ConsistentChooseKHasher::new(i + 32783, 2); + let hasher = hasher_for_key(i + 32783); + let sampler = ConsistentChooseKHasher::new(hasher, 2); let samples = sampler.prev(stats.len()); for s in samples { stats[s] += 1; @@ -274,8 +331,9 @@ mod tests { for k in 1..10 { for n in k + 1..20 { for key in 0..1000 { - let sampler1 = ConsistentChooseKHasher::new(key, k); - let sampler2 = ConsistentChooseKHasher::new(key, k + 1); + let hasher = hasher_for_key(key); + let sampler1 = ConsistentChooseKHasher::new(hasher.clone(), k); + let sampler2 = ConsistentChooseKHasher::new(hasher, k + 1); let set1 = sampler1.prev(n); let set2 = sampler2.prev(n); assert_eq!(set1.len(), k); From 0baaafc83b2b58f5023aea9e814170f01c5cc091 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 13 Aug 2025 15:55:34 +0200 Subject: [PATCH 10/20] Update lib.rs --- crates/consistent-hashing/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs index dd7675c..b7dc434 100644 --- a/crates/consistent-hashing/src/lib.rs +++ b/crates/consistent-hashing/src/lib.rs @@ -256,6 +256,8 @@ impl ConsistentChooseKHasher { #[cfg(test)] mod tests { + use std::hash::DefaultHasher; + use super::*; fn hasher_for_key(key: u64) -> DefaultHasher { From 0dcb137a20015a825ca4f5d833fb3af737c3a7aa Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Wed, 13 Aug 2025 17:51:45 +0200 Subject: [PATCH 11/20] Update README.md --- crates/consistent-hashing/README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index 4f29c56..b15224d 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -81,18 +81,37 @@ In the remainder of this section we prove that the `consistent_choose_k` algorit Let's define `M(k,n) = consistent_choose_max(_, k, n)` and `S(k, n) := consistent_choose_k(_, k, n)` as short-cuts for some arbitrary fixed `key`. We assume that `consistent_hash(key, k, n)` computes `k` independent consistent hash functions. +### Property 1 + Since `M(k, n) < n` and `S(k, n) = {M(k, n)} ∪ S(k - 1, M(k, n))` for `k > 1`, `S(k, n)` constructs a strictly monotonically decreasing sequence. The sequence outputs exactly `k` elements which therefore must all be distinct which proves property 1 for `k <= n`. Properties 2, 3, and 4 can be proven via induction as follows. +### Property 4 + `k = 1`: We expect that `consistent_hash` returns a single uniformly distributed node index which is consistent in `n`, i.e. changes the hash value with probability `1/(n+1)`, when `n` increments by one. In our implementation, we use an `O(1)` implementation of the jump-hash algorithm. For `k=1`, `consistent_choose_k(key, 1, n)` becomes a single function call to `consistent_choose_max(key, 1, n)` which in turn calls `consistent_hash(key, 0, n)`. I.e. `consistent_choose_k` inherits the all the desired properties from `consistent_hash` for `k=1` and all `n>=1`. `k → k+1`: `M(k+1, n+1) = M(k+1, n)` iff `M(k, n+1) < n` and `consistent_hash(_, k, n+1-k) < n - k`. The probability for this is `(n+1-k)/(n+1)` for the former by induction and `(n-k)/(n+1-k)` by the assumption that `consistent_hash` is a proper consistent hash function. Since both these probabilities are assumed to be independent, the probability that our initial value changes is `1 - (n+1-k)/(n+1) * (n-k)/(n+1-k) = 1 - (n-k)/(n+1) = (k+1)/(n+1)` proving property 4. +### Property 3 + Property 3 is trivially satisfied if `S(k+1, n+1) = S(k+1, n)`. So, we focus on the case where `S(k+1, n+1) != S(k+1, n)`, which implies that `n ∈ S(k+1, n+1)` as largest element. We know that `S(k+1, n) = {m} ∪ S(k, m)` for some `m` by definition and `S(k, n) = S(k, u) ∖ {v} ∪ {w}` by induction for some `u`, `v`, and `w`. Thus far we have `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, u) ∖ {v} ∪ {w}`. If `u = m`, then `S(k+1, n) = {m} ∪ S(k, m) ∖ {v} ∪ {w}` and `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, m) ∖ {v} ∪ {w}` and the two differ exaclty in the elemetns `m` and `n` proving property 3. If `u ≠ m`, then `consistent_hash(_, k, n) = m`, since that's the only way how the largest values in `S(k+1, n)` and `S(k, n)` can differ. In this case, `m ∉ S(k+1, n+1)`, since `n` (and not `m`) is the largest element of `S(k+1, n+1)`. Furthermore, `S(k, n) = S(k, m)`, since `consistent_hash(_, i, n) < m` for all `i < k` (otherwise there is a contradiction). -Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m} ∪ S(k, m)` which differ exactly in the elements `n` and `m` which concludes the proof. \ No newline at end of file +Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m} ∪ S(k, m)` which differ exactly in the elements `n` and `m` which concludes the proof. + +### Property 2 + +The final part is to prove property 2. This time we have an inducation over `k` and `n`. +As before, induction start for `k=1` and for all `n>0` is inherited from the `consistency_hash` implementation. The case `n=k` is also trivially covered, since the only valid set are the numbers `{0, ..., k-1}` which the algorithm correctly outputs. So, we only need to care about the induction step where `k>1` and `n>k`. + +We need to prove that `P(i ∈ S(k+1, n+1)) = (k+1)/(n+1)` for all `0 <= i <= n`. Property 3 already proves the case `i = n`. Furthermore we know that `P(n ∈ S(k+1, n+1)) = (k+1)/(n+1)` and vice versa `P(n ∉ S(k+1, n+1)) = 1 - (k+1)/(n+1)`. Let's consider those two cases separately. + +`n ∈ S(k+1, n+1)`: By the definition of `S`, we know that `S(k+1, n+1) = {n} ∪ S(k, n)`. `P(i ∈ S(k+1, n+1)) = P(i ∈ S(k, n)) P(n ∈ S(k+1, n+1)) = k/n * (k+1)/(n+1)` for all `0 <= i < n`. + +`n ∉ S(k+1, n+1)`: Once more by definition, `S(k+1, n+1) = S(k+1, n)` in this case. `P(i ∈ S(k+1, n+1)) = P(i ∈ S(k+1, n)) P(n ∉ S(k+1, n+1)) = (k+1)/n * (1 - (k+1)/(n+1))` for all `0 <= i < n`. + +Summing both cases together leads to `P(i ∈ S(k+1, n+1)) = k/n * (k+1)/(n+1) + (k+1)/n * (1 - (k+1)/(n+1)) = k/n * (k+1)/(n+1) + k/n * (1 - (k+1)/(n+1)) + 1/n * (1 - (k+1)/(n+1)) = k/n * (k+1)/(n+1) + k/n - k/n * (k+1)/(n+1) + 1/n - 1/n * (k+1)/(n+1) = k/n + 1/n - 1/n * (k+1)/(n+1) = (k+1)/n - (k+1)/(n+1)/n = (k+1)/n * (1 - 1/(n+1)) = (k+1)/(n+1)` for all `0 <= i < n` which concludes the proof. From d4b841024aa2ee72aec331033974b7519d71df9d Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 10:13:25 +0200 Subject: [PATCH 12/20] Update crates/consistent-hashing/README.md Co-authored-by: Luke Francl --- crates/consistent-hashing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index b15224d..7de40f2 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -59,7 +59,7 @@ fn consistent_hash(key: Key, i: usize, n: usize) -> usize { ``` `consistent_choose_k` makes `k` calls to `consistent_choose_max` which calls `consistent_hash` another `k` times. -In total, `consistent_hash` is called `k * (k+1) / 2` Utilizing a `O(1)` solution for `consistent_hash` leads to a `O(k^2)` runtime. +In total, `consistent_hash` is called `k * (k+1) / 2` many times. Utilizing a `O(1)` solution for `consistent_hash` leads to a `O(k^2)` runtime. This runtime can be further improved by replacing the max operation with a heap where popped elements are updated according to the new arguments `n` and `k`. With this optimization, the complexity reduces to `O(k log k)`. With some probabilistic bucketing strategy, it should be possible to reduce the expected runtime to `O(k)`. From 0935ea061da1bda16d942c29b39cd7904800312b Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 10:13:35 +0200 Subject: [PATCH 13/20] Update crates/consistent-hashing/README.md Co-authored-by: Luke Francl --- crates/consistent-hashing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index 7de40f2..8dfa1c3 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -98,7 +98,7 @@ Properties 2, 3, and 4 can be proven via induction as follows. Property 3 is trivially satisfied if `S(k+1, n+1) = S(k+1, n)`. So, we focus on the case where `S(k+1, n+1) != S(k+1, n)`, which implies that `n ∈ S(k+1, n+1)` as largest element. We know that `S(k+1, n) = {m} ∪ S(k, m)` for some `m` by definition and `S(k, n) = S(k, u) ∖ {v} ∪ {w}` by induction for some `u`, `v`, and `w`. Thus far we have `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, u) ∖ {v} ∪ {w}`. -If `u = m`, then `S(k+1, n) = {m} ∪ S(k, m) ∖ {v} ∪ {w}` and `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, m) ∖ {v} ∪ {w}` and the two differ exaclty in the elemetns `m` and `n` proving property 3. +If `u = m`, then `S(k+1, n) = {m} ∪ S(k, m) ∖ {v} ∪ {w}` and `S(k+1, n+1) = {n} ∪ S(k, n) = {n} ∪ S(k, m) ∖ {v} ∪ {w}` and the two differ exactly in the elemetns `m` and `n` proving property 3. If `u ≠ m`, then `consistent_hash(_, k, n) = m`, since that's the only way how the largest values in `S(k+1, n)` and `S(k, n)` can differ. In this case, `m ∉ S(k+1, n+1)`, since `n` (and not `m`) is the largest element of `S(k+1, n+1)`. Furthermore, `S(k, n) = S(k, m)`, since `consistent_hash(_, i, n) < m` for all `i < k` (otherwise there is a contradiction). Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m} ∪ S(k, m)` which differ exactly in the elements `n` and `m` which concludes the proof. From 99c69f3a059e09668f27b2c51a49bcb706c735f5 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 10:15:12 +0200 Subject: [PATCH 14/20] Update crates/consistent-hashing/README.md Co-authored-by: Luke Francl --- crates/consistent-hashing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index 8dfa1c3..fb3fda8 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -106,7 +106,7 @@ Putting it together leads to `S(k+1, n+1) = {n} ∪ S(k, m)` and `S(k+1, n) = {m ### Property 2 The final part is to prove property 2. This time we have an inducation over `k` and `n`. -As before, induction start for `k=1` and for all `n>0` is inherited from the `consistency_hash` implementation. The case `n=k` is also trivially covered, since the only valid set are the numbers `{0, ..., k-1}` which the algorithm correctly outputs. So, we only need to care about the induction step where `k>1` and `n>k`. +As before, the base case of the induction for `k=1` and all `n>0` is inherited from the `consistency_hash` implementation. The case `n=k` is also trivially covered, since the only valid set are the numbers `{0, ..., k-1}` which the algorithm correctly outputs. So, we only need to care about the induction step where `k>1` and `n>k`. We need to prove that `P(i ∈ S(k+1, n+1)) = (k+1)/(n+1)` for all `0 <= i <= n`. Property 3 already proves the case `i = n`. Furthermore we know that `P(n ∈ S(k+1, n+1)) = (k+1)/(n+1)` and vice versa `P(n ∉ S(k+1, n+1)) = 1 - (k+1)/(n+1)`. Let's consider those two cases separately. From 496f5394f0f96c455357cff2839e1d1554f69748 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 10:15:39 +0200 Subject: [PATCH 15/20] Update crates/consistent-hashing/README.md Co-authored-by: Luke Francl --- crates/consistent-hashing/README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index fb3fda8..fb95783 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -15,16 +15,15 @@ Common algorithms where `N` is the number of nodes and `R` is the number of replicas. -| Algorithm | Lookup per key | Node add/remove | Memory | Lookup with replication | -| | (no replication) | | | | -|-------------------------|---------------------|----------------------------------------|---------------------------|-------------------------------------| -| Hash ring (with vnodes) | O(log N): binary search over N points; O(1): with specialized structures | O(log N) | O(N) | O(log N + R): Take next R distinct successors | -| Rendezvous | O(N): max score | O(1) | O(N) node list | O(N log R): pick top R scores | -| Jump consistent hash | O(log(N)) expected | 0 | O(1) | O(R log N) | -| AnchorHash | O(1) expected | O(1) | O(N) | Not native | -| DXHash | O(1) expected | O(1) | O(N) | Not native | -| JumpBackHash | O(1) expected | 0 | O(1) | Not native | -| **ConsistentChooseK** | **O(1) expected** | **0** | **O(1)** | **O(R^2)**; **O(R log(R))**: using heap | +| Algorithm | Lookup per key
(no replication) | Node add/remove | Memory | Lookup with replication | +|-------------------------|--------------------------------------------------------------------------|-----------------|----------------|-----------------------------------------------| +| Hash ring (with vnodes) | O(log N): binary search over N points; O(1): with specialized structures | O(log N) | O(N) | O(log N + R): Take next R distinct successors | +| Rendezvous | O(N): max score | O(1) | O(N) node list | O(N log R): pick top R scores | +| Jump consistent hash | O(log(N)) expected | 0 | O(1) | O(R log N) | +| AnchorHash | O(1) expected | O(1) | O(N) | Not native | +| DXHash | O(1) expected | O(1) | O(N) | Not native | +| JumpBackHash | O(1) expected | 0 | O(1) | Not native | +| **ConsistentChooseK** | **O(1) expected** | **0** | **O(1)** | **O(R^2)**; **O(R log(R))**: using heap | Replication of keys - Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas more evenly. Replicas are not independently distributed. From 23f308089d61b95e3535d3f133a015ad4f91d82a Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 14:05:38 +0200 Subject: [PATCH 16/20] add benchmark --- Cargo.toml | 1 + .../consistent-hashing/benchmarks/Cargo.toml | 15 +++++ .../benchmarks/criterion.toml | 18 ++++++ .../benchmarks/performance.rs | 63 +++++++++++++++++++ crates/consistent-hashing/src/lib.rs | 6 +- 5 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 crates/consistent-hashing/benchmarks/Cargo.toml create mode 100644 crates/consistent-hashing/benchmarks/criterion.toml create mode 100644 crates/consistent-hashing/benchmarks/performance.rs diff --git a/Cargo.toml b/Cargo.toml index 312f46d..0b09dcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/*", "crates/bpe/benchmarks", "crates/bpe/tests", + "crates/consistent-hashing/benchmarks", ] resolver = "2" diff --git a/crates/consistent-hashing/benchmarks/Cargo.toml b/crates/consistent-hashing/benchmarks/Cargo.toml new file mode 100644 index 0000000..580e5ab --- /dev/null +++ b/crates/consistent-hashing/benchmarks/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "consistent-hashing-benchmarks" +edition = "2021" + +[[bench]] +name = "performance" +path = "performance.rs" +harness = false +test = false + +[dependencies] +consistent-hashing = { path = "../" } + +criterion = { version = "0.7", features = ["csv_output"] } +rand = "0.9" diff --git a/crates/consistent-hashing/benchmarks/criterion.toml b/crates/consistent-hashing/benchmarks/criterion.toml new file mode 100644 index 0000000..0e43927 --- /dev/null +++ b/crates/consistent-hashing/benchmarks/criterion.toml @@ -0,0 +1,18 @@ +# save report in this directory, even if a custom target directory is set +criterion_home = "./target/criterion" + +# The colors table allows users to configure the colors used by the charts +# cargo-criterion generates. +[colors] +# Color-blind friendly color scheme from https://personal.sron.nl/~pault/. +comparison_colors = [ + {r = 51, g = 34, b = 136 }, # indigo + {r = 136, g = 204, b = 238 }, # cyan + {r = 68, g = 170, b = 153 }, # teal + {r = 17, g = 119, b = 51 }, # green + {r = 153, g = 153, b = 51 }, # olive + {r = 221, g = 204, b = 119 }, # sand + {r = 204, g = 102, b = 119 }, # rose + {r = 136, g = 34, b = 85 }, # wine + {r = 170, g = 68, b = 153 }, # purple +] diff --git a/crates/consistent-hashing/benchmarks/performance.rs b/crates/consistent-hashing/benchmarks/performance.rs new file mode 100644 index 0000000..4bdc91b --- /dev/null +++ b/crates/consistent-hashing/benchmarks/performance.rs @@ -0,0 +1,63 @@ +use std::{ + hash::{DefaultHasher, Hash}, + hint::black_box, + time::Duration, +}; + +use consistent_hashing::{ConsistentChooseKHasher, ConsistentHasher}; +use criterion::{ + criterion_group, criterion_main, AxisScale, Bencher, BenchmarkId, Criterion, PlotConfiguration, + Throughput, +}; +use rand::{rng, Rng}; + +fn throughput_benchmark(c: &mut Criterion) { + let keys: Vec = rng().random_iter().take(1000).collect(); + + let mut group = c.benchmark_group(format!("choose")); + group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic)); + for n in [1usize, 10, 100, 1000, 10000] { + group.throughput(Throughput::Elements(keys.len() as u64)); + group.bench_with_input(BenchmarkId::new(format!("1"), n), &n, |b, n| { + b.iter_batched( + || &keys, + |keys| { + for key in keys { + let mut h = DefaultHasher::new(); + key.hash(&mut h); + black_box(ConsistentHasher::new(h).prev(*n + 1)); + } + }, + criterion::BatchSize::SmallInput, + ) + }); + for k in [1, 2, 3, 10, 100] { + group.bench_with_input(BenchmarkId::new(format!("k_{k}"), n), &n, |b, n| { + b.iter_batched( + || &keys, + |keys| { + let mut res = Vec::with_capacity(k); + for key in keys { + let mut h = DefaultHasher::new(); + key.hash(&mut h); + black_box(ConsistentChooseKHasher::new(h, k).prev(*n + k, &mut res)); + } + }, + criterion::BatchSize::SmallInput, + ) + }); + } + } + group.finish(); +} + +criterion_group!( + name = benches; + config = Criterion::default() + .warm_up_time(Duration::from_millis(500)) + .measurement_time(Duration::from_millis(4000)) + .nresamples(1000); + + targets = throughput_benchmark, +); +criterion_main!(benches); diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs index b7dc434..7a19eb8 100644 --- a/crates/consistent-hashing/src/lib.rs +++ b/crates/consistent-hashing/src/lib.rs @@ -232,11 +232,11 @@ impl ConsistentChooseKHasher { } // TODO: Implement this as an iterator! - pub fn prev(&self, mut n: usize) -> Vec { - let mut samples = Vec::with_capacity(self.k); + pub fn prev(&self, mut n: usize, samples: &mut Vec) { let mut samplers: Vec<_> = (0..self.k) .map(|i| ConsistentHashRevIterator::new(n - i, self.builder.seq_builder(i)).peekable()) .collect(); + samples.clear(); for i in (0..self.k).rev() { let mut max = 0; for k in 0..=i { @@ -248,8 +248,6 @@ impl ConsistentChooseKHasher { samples.push(max); n = max; } - samples.sort(); - samples } } From 5d522374057df648bba7e3aee1b3276415788255 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 14:38:23 +0200 Subject: [PATCH 17/20] remove second vector --- .../benchmarks/performance.rs | 10 ++-- crates/consistent-hashing/src/lib.rs | 57 +++++++++++++------ 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/crates/consistent-hashing/benchmarks/performance.rs b/crates/consistent-hashing/benchmarks/performance.rs index 4bdc91b..8dc8e62 100644 --- a/crates/consistent-hashing/benchmarks/performance.rs +++ b/crates/consistent-hashing/benchmarks/performance.rs @@ -1,12 +1,12 @@ use std::{ - hash::{DefaultHasher, Hash}, + hash::{DefaultHasher, Hash, Hasher}, hint::black_box, time::Duration, }; use consistent_hashing::{ConsistentChooseKHasher, ConsistentHasher}; use criterion::{ - criterion_group, criterion_main, AxisScale, Bencher, BenchmarkId, Criterion, PlotConfiguration, + criterion_group, criterion_main, AxisScale, BenchmarkId, Criterion, PlotConfiguration, Throughput, }; use rand::{rng, Rng}; @@ -23,7 +23,7 @@ fn throughput_benchmark(c: &mut Criterion) { || &keys, |keys| { for key in keys { - let mut h = DefaultHasher::new(); + let mut h = DefaultHasher::default(); key.hash(&mut h); black_box(ConsistentHasher::new(h).prev(*n + 1)); } @@ -38,9 +38,9 @@ fn throughput_benchmark(c: &mut Criterion) { |keys| { let mut res = Vec::with_capacity(k); for key in keys { - let mut h = DefaultHasher::new(); + let mut h = DefaultHasher::default(); key.hash(&mut h); - black_box(ConsistentChooseKHasher::new(h, k).prev(*n + k, &mut res)); + black_box(ConsistentChooseKHasher::new(h, k).prev_with_vec(*n + k, &mut res)); } }, criterion::BatchSize::SmallInput, diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs index 7a19eb8..fd6391f 100644 --- a/crates/consistent-hashing/src/lib.rs +++ b/crates/consistent-hashing/src/lib.rs @@ -12,6 +12,7 @@ pub trait HashSequence { pub trait HashSeqBuilder { type Seq: HashSequence; + /// Returns a bit mask indicating which buckets have at least one hash. fn bit_mask(&self) -> u64; /// Return a HashSequence instance which is seeded with the given bit position /// and the seed of this builder. @@ -66,7 +67,7 @@ struct BucketIterator { hasher: H, n: usize, is_first: bool, - bit: u64, // A bitmask with a single bit set. + bit: u64, // A bitmask with a single bit set. } impl BucketIterator { @@ -199,20 +200,30 @@ pub struct ConsistentHasher { builder: H, } -impl ConsistentHasher { +impl ConsistentHasher { pub fn new(builder: H) -> Self { Self { builder } } - pub fn prev(&self, n: usize) -> Option { + pub fn prev(&self, n: usize) -> Option + where + H: Clone, + { let mut sampler = ConsistentHashRevIterator::new(n, self.builder.clone()); sampler.next() } - pub fn next(&self, n: usize) -> Option { + pub fn next(&self, n: usize) -> Option + where + H: Clone, + { let mut sampler = ConsistentHashIterator::new(n, self.builder.clone()); sampler.next() } + + pub fn into_prev(self, n: usize) -> Option { + ConsistentHashRevIterator::new(n, self.builder).next() + } } /// Implementation of a consistent choose k hashing algorithm. @@ -231,27 +242,38 @@ impl ConsistentChooseKHasher { Self { builder, k } } - // TODO: Implement this as an iterator! - pub fn prev(&self, mut n: usize, samples: &mut Vec) { - let mut samplers: Vec<_> = (0..self.k) - .map(|i| ConsistentHashRevIterator::new(n - i, self.builder.seq_builder(i)).peekable()) - .collect(); + pub fn prev(&self, n: usize) -> Vec { + let mut res = Vec::with_capacity(self.k); + self.prev_with_vec(n, &mut res); + res + } + + pub fn prev_with_vec(&self, mut n: usize, samples: &mut Vec) { + assert!(n >= self.k, "n must be at least k"); samples.clear(); + for i in 0..self.k { + samples.push( + ConsistentHasher::new(self.builder.seq_builder(i)) + .into_prev(n - i) + .expect("must not fail") + + i, + ); + } for i in (0..self.k).rev() { - let mut max = 0; - for k in 0..=i { - while samplers[k].peek() >= Some(&(n - k)) && n - k > 0 { - samplers[k].next(); + n = samples[0..=i].iter().copied().max().expect(""); + samples[i] = n; + for j in 0..i { + if samples[j] == n { + samples[j] = ConsistentHasher::new(self.builder.seq_builder(j)) + .into_prev(n - j) + .expect("must not fail") + + j; } - max = max.max(samplers[k].peek().unwrap() + k); } - samples.push(max); - n = max; } } } - #[cfg(test)] mod tests { use std::hash::DefaultHasher; @@ -327,6 +349,7 @@ mod tests { } } println!("{stats:?}"); + assert_eq!(stats, vec![10, 12, 6, 6, 6, 5, 9, 10]); // Test consistency when increasing k! for k in 1..10 { for n in k + 1..20 { From f6e29f7faa428345548868c66135d904fdf33c82 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 15:16:51 +0200 Subject: [PATCH 18/20] Update README.md --- crates/consistent-hashing/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/consistent-hashing/README.md b/crates/consistent-hashing/README.md index fb95783..0830c66 100644 --- a/crates/consistent-hashing/README.md +++ b/crates/consistent-hashing/README.md @@ -28,8 +28,8 @@ where `N` is the number of nodes and `R` is the number of replicas. Replication of keys - Hash ring: replicate by walking clockwise to the next R distinct nodes. Virtual nodes help spread replicas more evenly. Replicas are not independently distributed. - Rendezvous hashing: replicate by selecting the top R nodes by score for the key. This naturally yields R distinct owners and supports weights. -- Jump consistent hash: the base function doesn't support replication. But the math can be easily modified to support consistent replication. -- JumpBackHash and variants: The trick of Jump consistent hash to support replication won't work here due to the additional state introduced. +- Jump consistent hash: the base function doesn't support replication. While the math can be modified to support consistent replication, it cannot be efficiently solved for large k and even for small k (=2 or =3), a quadratic or cubic equation has to be solved. +- JumpBackHash and variants: The trick of Jump consistent hash to support replication won't work here due to the introduction of additional state. - ConsistentChooseK: Faster and more memory efficient than all other solutions. Why replication matters From d20f9b600a8704245ef4e65154d6d51767cbf832 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 15:17:59 +0200 Subject: [PATCH 19/20] Update performance.rs --- crates/consistent-hashing/benchmarks/performance.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/consistent-hashing/benchmarks/performance.rs b/crates/consistent-hashing/benchmarks/performance.rs index 8dc8e62..05dd929 100644 --- a/crates/consistent-hashing/benchmarks/performance.rs +++ b/crates/consistent-hashing/benchmarks/performance.rs @@ -40,7 +40,9 @@ fn throughput_benchmark(c: &mut Criterion) { for key in keys { let mut h = DefaultHasher::default(); key.hash(&mut h); - black_box(ConsistentChooseKHasher::new(h, k).prev_with_vec(*n + k, &mut res)); + black_box( + ConsistentChooseKHasher::new(h, k).prev_with_vec(*n + k, &mut res), + ); } }, criterion::BatchSize::SmallInput, From 1dde97c913c18af5e992fd633e2d9f94d82e2838 Mon Sep 17 00:00:00 2001 From: Alexander Neubeck Date: Fri, 15 Aug 2025 15:22:38 +0200 Subject: [PATCH 20/20] make linter happy --- crates/consistent-hashing/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/consistent-hashing/src/lib.rs b/crates/consistent-hashing/src/lib.rs index fd6391f..dcfbb0c 100644 --- a/crates/consistent-hashing/src/lib.rs +++ b/crates/consistent-hashing/src/lib.rs @@ -262,9 +262,9 @@ impl ConsistentChooseKHasher { for i in (0..self.k).rev() { n = samples[0..=i].iter().copied().max().expect(""); samples[i] = n; - for j in 0..i { - if samples[j] == n { - samples[j] = ConsistentHasher::new(self.builder.seq_builder(j)) + for (j, sample) in samples[0..i].iter_mut().enumerate() { + if *sample == n { + *sample = ConsistentHasher::new(self.builder.seq_builder(j)) .into_prev(n - j) .expect("must not fail") + j;