写点什么

五、一致性哈希算法

用户头像
Geek_28b526
关注
发布于: 2020 年 11 月 20 日

简单实现一致性哈希算法


Server


public class Server { private String serverName; //1:up, 2:down private volatile int status = 1; public String getServerName() { return serverName; } public void setServerName(String serverName) { this.serverName = serverName; } public int getStatus() { return status; } public void setStatus(int status) { this.status = status; } public boolean isAvailable(){ return this.status == 1; } @Override public String toString() { return serverName; } }
复制代码


ShardingAlgo,简单实现一致性哈希算法

import java.util.List;import java.util.TreeMap;import java.util.Map.Entry;
public class ShardingAlgo { private TreeMap<Long, Server> nodes; private final MurmurHash algo = new MurmurHash(); public ShardingAlgo(List<Server> servers, int virtualNodeNum){ initialize(servers, virtualNodeNum); } private void initialize(List<Server> servers, int virtualNodeNum) { nodes = new TreeMap<Long, Server>();
for (int i = 0; i < servers.size(); i++) { Server server = servers.get(i); for (int n = 0; n < virtualNodeNum; n++) { nodes.put(this.algo.hash("SHARD-" + i + "-NODE-" + n), server); } } } public Server getShardInfo2(String key){ Entry<Long, Server> entry = select(algo.hash(key)); /* while(!entry.getValue().isAvailable()){ entry = select(entry.getKey() + 1); } */ return entry.getValue(); } private Entry<Long, Server> select(Long hashVal){ Entry<Long, Server> entry = nodes.ceilingEntry(hashVal); if(entry == null){ entry = nodes.firstEntry(); } return entry; }}
复制代码


对于 key 的 hash, 采用 MurmurHash 算法

/* * Licensed to the Apache Software Foundation (ASF) under one or more contributor license * agreements. See the NOTICE file distributed with this work for additional information regarding * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License * for the specific language governing permissions and limitations under the License. */
import java.io.UnsupportedEncodingException;import java.nio.ByteBuffer;import java.nio.ByteOrder;
/** * This is a very fast, non-cryptographic hash suitable for general hash-based lookup. See * http://murmurhash.googlepages.com/ for more details. <br> * <p> * The C version of MurmurHash 2.0 found at that site was ported to Java by Andrzej Bialecki (ab at * getopt org). * </p> */public class MurmurHash { /** * Hashes bytes in an array. * @param data The bytes to hash. * @param seed The seed for the hash. * @return The 32 bit hash of the bytes in question. */ public static int hash(byte[] data, int seed) { return hash(ByteBuffer.wrap(data), seed); }
/** * Hashes bytes in part of an array. * @param data The data to hash. * @param offset Where to start munging. * @param length How many bytes to process. * @param seed The seed to start with. * @return The 32-bit hash of the data in question. */ public static int hash(byte[] data, int offset, int length, int seed) { return hash(ByteBuffer.wrap(data, offset, length), seed); }
/** * Hashes the bytes in a buffer from the current position to the limit. * @param buf The bytes to hash. * @param seed The seed for the hash. * @return The 32 bit murmur hash of the bytes in the buffer. */ public static int hash(ByteBuffer buf, int seed) { // save byte order for later restoration ByteOrder byteOrder = buf.order(); buf.order(ByteOrder.LITTLE_ENDIAN);
int m = 0x5bd1e995; int r = 24;
int h = seed ^ buf.remaining();
int k; while (buf.remaining() >= 4) { k = buf.getInt();
k *= m; k ^= k >>> r; k *= m;
h *= m; h ^= k; }
if (buf.remaining() > 0) { ByteBuffer finish = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); // for big-endian version, use this first: // finish.position(4-buf.remaining()); finish.put(buf).rewind(); h ^= finish.getInt(); h *= m; }
h ^= h >>> 13; h *= m; h ^= h >>> 15;
buf.order(byteOrder); return h; }
public static long hash64A(byte[] data, int seed) { return hash64A(ByteBuffer.wrap(data), seed); }
public static long hash64A(byte[] data, int offset, int length, int seed) { return hash64A(ByteBuffer.wrap(data, offset, length), seed); }
public static long hash64A(ByteBuffer buf, int seed) { ByteOrder byteOrder = buf.order(); buf.order(ByteOrder.LITTLE_ENDIAN);
long m = 0xc6a4a7935bd1e995L; int r = 47;
long h = seed ^ (buf.remaining() * m);
long k; while (buf.remaining() >= 8) { k = buf.getLong();
k *= m; k ^= k >>> r; k *= m;
h ^= k; h *= m; }
if (buf.remaining() > 0) { ByteBuffer finish = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); // for big-endian version, do this first: // finish.position(8-buf.remaining()); finish.put(buf).rewind(); h ^= finish.getLong(); h *= m; }
h ^= h >>> r; h *= m; h ^= h >>> r;
buf.order(byteOrder); return h; }
public long hash(byte[] key) { return hash64A(key, 0x1234ABCD); }
public long hash(String key){ try { return hash(key.getBytes("UTF-8")); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } }}
复制代码

测试 ShardingTest

import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;

public class ShardingTest {
public static void main(String[] args) { //10台服务器 List<Server> servers = new ArrayList<Server>(); for(int i = 0; i < 10; i++){ Server server = new Server(); server.setServerName("server"+i); server.setStatus(1); servers.add(server); } //n个不同虚拟节点node int[] nodeNums = {1, 10, 50, 100, 160, 512, 1024, 2048, 5120, 6144, 7168}; //样本数 int sampleSize = 1000000; batch(servers, nodeNums, sampleSize); } public static void batch(List<Server> servers, int[] virtualNodeNum, int sampleSize){ for(int n = 0; n < virtualNodeNum.length; n++){ ShardingAlgo sa = new ShardingAlgo(servers, virtualNodeNum[n]); //100万样本 Map<Server, Long> s = new HashMap<Server, Long>(); Server server; for(int i = 0; i < sampleSize; i++){ server = sa.getShardInfo2("key"+i); s.put(server, s.get(server) == null ? 1 : s.get(server) + 1); } System.out.println("虚拟节点数量:"+virtualNodeNum[n]); System.out.println("请求分布情况:"+s); Long[] arrays = new Long[s.values().size()]; s.values().toArray(arrays); SimpleMath simpleMath = new SimpleMath(); double sd = simpleMath.standardDiviation(arrays); System.out.println("sd:"+sd); } } public static class SimpleMath {
//标准差σ=sqrt(s^2) public double standardDiviation(Long[] x) { int m=x.length; double sum=0; for(int i=0;i<m;i++){//求和 sum+=x[i]; } double dAve=sum/m;//求平均值 double dVar=0; for(int i=0;i<m;i++){//求方差 dVar+=(x[i]-dAve)*(x[i]-dAve); } return Math.sqrt(dVar/m);
} }}
复制代码


从测试结果可以看出,虚拟节点数量越多分布越均衡

虚拟节点数量:1

请求分布情况:{server9=226728, server7=172926, server5=59631, server2=621, server8=38234, server3=381636, server4=6560, server0=33260, server1=7972, server6=72432}

sd:117657.26204616527


虚拟节点数量:10

请求分布情况:{server9=96367, server7=129819, server5=99148, server8=106359, server2=92083, server3=165211, server4=67579, server0=86523, server6=101213, server1=55698}

sd:29078.474836208312


虚拟节点数量:50

请求分布情况:{server9=88068, server6=93234, server3=89246, server1=82645, server8=129902, server2=107232, server0=88398, server4=89044, server5=103617, server7=128614}

sd:16241.6482353239


虚拟节点数量:100

请求分布情况:{server9=113131, server6=96687, server3=93544, server8=92400, server1=97450, server2=104788, server0=110036, server4=93022, server5=106223, server7=92719}

sd:7439.213533700992


虚拟节点数量:160

请求分布情况:{server9=106698, server6=102617, server8=103252, server3=96098, server1=101420, server2=102120, server0=102192, server4=98627, server5=93210, server7=93766}

sd:4191.42756110612


虚拟节点数量:512

请求分布情况:{server9=104081, server6=101755, server3=100441, server8=98932, server1=102022, server2=101299, server0=92397, server4=99558, server5=100278, server7=99237}

sd:2924.140933676077


虚拟节点数量:1024

请求分布情况:{server9=97005, server6=96974, server3=100042, server1=102631, server8=103432, server2=100050, server0=96467, server4=102380, server5=96065, server7=104954}

sd:3082.2812979999085


虚拟节点数量:2048

请求分布情况:{server9=97631, server6=99021, server3=99505, server1=99368, server8=103967, server2=100414, server0=95851, server4=102824, server5=99944, server7=101475}

sd:2247.0307964066715


虚拟节点数量:5120

请求分布情况:{server9=100186, server6=100667, server3=100130, server8=101199, server1=98329, server2=101509, server0=98115, server4=99238, server5=101084, server7=99543}

sd:1118.9764072579903


虚拟节点数量:6144

请求分布情况:{server9=100563, server6=99866, server3=101001, server8=100369, server1=99340, server2=101196, server0=98652, server4=99209, server5=99878, server7=99926}

sd:761.724884718886


虚拟节点数量:7168

请求分布情况:{server9=99226, server6=99370, server3=99422, server8=100527, server1=99405, server2=100663, server0=100706, server4=99631, server5=100149, server7=100901}

sd:622.0917938696829


用户头像

Geek_28b526

关注

还未添加个人签名 2020.02.26 加入

还未添加个人简介

评论

发布
暂无评论
五、一致性哈希算法