第五章作业

发布于: 2020 年 07 月 09 日

  • 用你熟悉的编程语言实现一致性 hash 算法。

  • 编写测试用例测试这个算法,测试 100 万 KV 数据,10 个服务器节点的情况下,计算这些 KV 数据在服务器上分布数量的标准差,以评估算法的存储负载不均衡性。

最优虚拟节点数:930,标准差:5153.4079209781175

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.lang3.StringUtils;
import java.util.*;
import java.util.stream.Collectors;
public class ConsistentHash {
//真实节点列表
private List<String> realNodes = new LinkedList<>();
//虚拟节点列表
private SortedMap<Integer, String> sortedMap = new TreeMap<Integer, String>();
private Map<String, Integer> realNodeKeyNum = Maps.newHashMap();
/**
* 初始化
*
* @param servers 待添加入Hash环的服务器列表
* @param virtualNodeNum 虚拟节点数量
*/
private void init(String[] servers, int virtualNodeNum) {
//添加真实节点
realNodeKeyNum = Maps.newHashMap();
for (int i = 0; i < servers.length; i++) {
realNodes.add(servers[i]);
realNodeKeyNum.put(servers[i], 0);
}
//添加虚拟节点
for (String str : realNodes) {
for (int i = 1; i <= virtualNodeNum; i++) {
String nodeName = str + "VM" + String.valueOf(i);
int hash = getHash(nodeName);
sortedMap.put(hash, nodeName);
// System.out.println("虚拟节点hash:" + hash + "【" + nodeName + "】放入");
}
}
}
//得到应当路由到的结点
private String getServer(String key) {
//得到该key的hash值
int hash = getHash(key);
//得到大于该Hash值的所有Map
String host;
SortedMap<Integer, String> subMap = sortedMap.tailMap(hash);
if (subMap.isEmpty()) {
//如果没有比该key的hash值大的,则从第一个node开始
Integer i = sortedMap.firstKey();
//返回对应的服务器
host = sortedMap.get(i);
} else {
//第一个Key就是顺时针过去离node最近的那个结点
Integer i = subMap.firstKey();
//返回对应的服务器
host = subMap.get(i);
}
if (StringUtils.isNotBlank(host)) {
String realHost = host.substring(0, host.indexOf("VM"));
// System.out.println(realHost);
return realHost;
}
return null;
}
//使用FNV1_32_HASH算法计算服务器的Hash值
private int getHash(String str) {
// int hash = str.hashCode();
final int p = 16777619;
int hash = (int) 2166136261L;
for (int i = 0; i < str.length(); i++) {
hash = (hash ^ str.charAt(i)) * p;
}
hash += hash << 13;
hash ^= hash >> 7;
hash += hash << 3;
hash ^= hash >> 17;
hash += hash << 5;
// 如果算出来的值为负数则取其绝对值
if (hash < 0) {
hash = Math.abs(hash);
}
return hash;
}
//计算key路由到的节点,计数+1
public void calcHash(String key) {
String server = getServer(key);
realNodeKeyNum.put(server, realNodeKeyNum.get(server) + 1);
}
//打印分配到各真实节点的key数量
public void print() {
List<Integer> countList = Lists.newArrayList();
realNodeKeyNum.entrySet().forEach(entrySet -> {
System.out.println(entrySet.getKey() + ":" + entrySet.getValue());
countList.add(entrySet.getValue());
});
System.out.println("标准差:" + Math.sqrt(listVariance(countList)));
}
/**
* 各节点分配到的key数量
* @return
*/
public List<Integer> getKeyNums() {
return realNodeKeyNum.values().stream().collect(Collectors.toList());
}
/**
* 计算集合的和
*/
public final static Double listSum(List<?> list) {
Double sum = new Double(0);
for (Object o : list) {
if (o instanceof Integer) {
Integer number = (Integer) o;
sum += number;
} else {
Double number = (Double) o;
sum += number;
}
}
return sum;
}
/**
* 计算集合的平均值
*/
public final static Double listAvg(List<?> list) {
return listSum(list) / list.size();
}
/**
* 计算集合的方差
*/
//方差s^2=[(x1-x)^2 +...(xn-x)^2]/n
public final static Double listVariance(List<?> list) {
int size = list.size();
double avg = listAvg(list);
double variance = 0;
for (Object o : list) {
if (o instanceof Integer) {
Integer number = (Integer) o;
variance += (number - avg) * (number - avg);
} else {
Double number = (Double) o;
variance += (number - avg) * (number - avg);
}
}
return variance / size;
}
/**
* 计算标准差 标准差=方差开平方根
*/
public final static Double listStandardDiviation(List<?> list) {
return Math.sqrt(listVariance(list));
}
public static void main(String[] args) {
int totalKeyNum = 1000000;
String[] servers = new String[]{"192.168.0.1:8080", "192.168.0.2:8080", "192.168.0.3:8080", "192.168.0.4:8080", "192.168.0.5:8080",
"192.168.0.6:8080", "192.168.0.7:8080", "192.168.0.8:8080", "192.168.0.9:8080", "192.168.0.10:8080"};
double minStandardDeviation = 999999;
int minStandardDeviationVirtualNodeNum = 0;
for (int virtualNodeNum = 10; virtualNodeNum <= 1000; virtualNodeNum += 10) {
ConsistentHash consistentHash = new ConsistentHash();
consistentHash.init(servers, 500);
for (int i = 0; i < totalKeyNum; i++) {
consistentHash.calcHash(UUID.randomUUID().toString());
}
List<Integer> keyNums = consistentHash.getKeyNums();
double sqrt = Math.sqrt(listVariance(keyNums));
if (minStandardDeviation > sqrt) {
minStandardDeviation = sqrt;
minStandardDeviationVirtualNodeNum = virtualNodeNum;
}
System.out.println("虚拟节点数:" + virtualNodeNum + ",标准差:" + sqrt);
}
System.out.println("===最优虚拟节点数:" + minStandardDeviationVirtualNodeNum + ",标准差:" + minStandardDeviation);
}
}

用户头像

小胖子

关注

还未添加个人签名 2018.02.04 加入

还未添加个人简介

评论

发布
暂无评论
第五章作业