Kubernetes 节点弹性扩展实践组件 Amazon Karpenter：部署 GPU 推理应用

export CLUSTER_NAME="karpenter-cluster"export AWS_REGION="us-west-2"export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)"

cat << EOF > cluster.yamlapiVersion: eksctl.io/v1alpha5kind: ClusterConfig
metadata: name: ${CLUSTER_NAME} region: ${AWS_REGION} version: "1.21" tags:   karpenter.sh/discovery: ${CLUSTER_NAME}
iam: withOIDC: true
managedNodeGroups: - name: ng-1   privateNetworking: true   instanceType: m5.large   desiredCapacity: 3EOF

export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output text)"

TEMPOUT=$(mktemp)curl -fsSL https://karpenter.sh/v0.6.3/getting-started/cloudformation.yaml > $TEMPOUT \&& aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ --capabilities CAPABILITY_NAMED_IAM \ --parameter-overrides "ClusterName=${CLUSTER_NAME}"

eksctl create iamidentitymapping \ --username system:node:{{EC2PrivateDNSName}} \ --cluster "${CLUSTER_NAME}" \ --arn "arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}" \ --group system:bootstrappers \ --group system:nodes

eksctl utils associate-iam-oidc-provider --cluster ${CLUSTER_NAME} –approveeksctl create iamserviceaccount \ --cluster "${CLUSTER_NAME}" --name karpenter --namespace karpenter \ --role-name "${CLUSTER_NAME}-karpenter" \ --attach-policy-arn "arn:aws:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME}" \ --role-only \ --approve

export KARPENTER_IAM_ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter"

helm repo add karpenter https://charts.karpenter.sh/helm repo updatehelm upgrade --install --namespace karpenter --create-namespace \ karpenter karpenter/karpenter \ --version v0.6.3 \ --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ --set clusterName=${CLUSTER_NAME} \ --set clusterEndpoint=${CLUSTER_ENDPOINT} \ --set aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ --set logLevel=debug \ --wait

cat << EOF > provisioner.yamlapiVersion: karpenter.sh/v1alpha5kind: Provisionermetadata: name: gpuspec: requirements:   - key: karpenter.sh/capacity-type     operator: In     values: ["on-demand"]    - key: node.kubernetes.io/instance-type     operator: In     values: ["g4dn.xlarge", "g4dn.2xlarge"] taints:   - key: nvidia.com/gpu     effect: "NoSchedule" limits:   resources:     gpu: 100 provider:   subnetSelector:     karpenter.sh/discovery: karpenter-cluster     securityGroupSelector:     kubernetes.io/cluster/karpenter-cluster: owned   ttlSecondsAfterEmpty: 30EOF

curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.0/docs/install/iam_policy.json
 aws iam create-policy \ --policy-name AWSLoadBalancerControllerIAMPolicy \ --policy-document file://iam_policy.json
 eksctl create iamserviceaccount \ --cluster=${CLUSTER_NAME} \ --namespace=kube-system \ --name=aws-load-balancer-controller \ --attach-policy-arn=arn:aws:iam::${AWS_ACCOUNT_ID}:policy/AWSLoadBalancerControllerIAMPolicy \ --override-existing-serviceaccounts \ --approve
 helm repo add eks https://aws.github.io/eks-charts helm repo update helm install aws-load-balancer-controller eks/aws-load-balancer-controller \ -n kube-system \ --set clusterName=${CLUSTER_NAME} \ --set serviceAccount.create=false \ --set serviceAccount.name=aws-load-balancer-controller

cat << EOF > resnet.yamlapiVersion: apps/v1kind: Deploymentmetadata: name: resnetspec: replicas: 1 selector:   matchLabels:     app: resnet-server template:   metadata:     labels:       app: resnet-server   spec:     # hostIPC is required for MPS communication     hostIPC: true     containers:     - name: resnet-container       image: seedjeffwan/tensorflow-serving-gpu:resnet       args:       - --per_process_gpu_memory_fraction=0.2       env:       - name: MODEL_NAME         value: resnet       ports:       - containerPort: 8501       # Use gpu resource here       resources:         requests:           nvidia.com/gpu: 1         limits:           nvidia.com/gpu: 1       volumeMounts:       - name: nvidia-mps         mountPath: /tmp/nvidia-mps     volumes:     - name: nvidia-mps       hostPath:         path: /tmp/nvidia-mps     tolerations:     - key: nvidia.com/gpu       effect: "NoSchedule"---apiVersion: v1kind: Servicemetadata: name: resnet-service annotations:   service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing   service.beta.kubernetes.io/aws-load-balancer-type: external   service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ipspec: type: LoadBalancer selector:   app: resnet-server ports: - port: 8501   targetPort: 8501EOF

from __future__ import print_function
import base64import requestsimport sys
assert (len(sys.argv) == 2), "Usage: resnet_client.py SERVER_URL"# The server URL specifies the endpoint of your server running the ResNet# model with the name "resnet" and using the predict interface.SERVER_URL = f'http://{sys.argv[1]}:8501/v1/models/resnet:predict'# The image URL is the location of the image we should send to the serverIMAGE_URL = 'https://tensorflow.org/images/blogs/serving/cat.jpg'
def main(): # Download the image dl_request = requests.get(IMAGE_URL, stream=True) dl_request.raise_for_status()
 # Compose a JSON Predict request (send JPEG image in base64). jpeg_bytes = base64.b64encode(dl_request.content).decode('utf-8') predict_request = '{"instances" : [{"b64": "%s"}]}' % jpeg_bytes
 # Send few requests to warm-up the model. for _ in range(3):   response = requests.post(SERVER_URL, data=predict_request)   response.raise_for_status()
 # Send few actual requests and report average latency. total_time = 0 num_requests = 10 for _ in range(num_requests):   response = requests.post(SERVER_URL, data=predict_request)   response.raise_for_status()   total_time += response.elapsed.total_seconds()   prediction = response.json()['predictions'][0]
 print('Prediction class: {}, avg latency: {} ms'.format(     prediction['classes'], (total_time*1000)/num_requests))
if __name__ == '__main__': main()

py $(kubectl get svc resnet-service -o=jsonpath='{.status.loadBalancer.ingress[0].hostname}')

kubectl 日志 -f -n karpenter -l app.kubernetes.io/name=karpenter -c 控制器