APISIX 可观测性最佳实践

traceType: 'w3c_traceparent'

{     "additional_attributes": [         "resp_body",         "balancer_ip",         "route_name"     ],    "disable": false,     "sampler": {         "name": "always_on"     } }

plugins: - opentelemetryplugin_attr:   opentelemetry:     resource:       service.name: APISIX       tenant.id: business_id     collector:       address: 127.0.0.1:9529/otel       request_timeout: 3       #request_headers:       # foo: bar     batch_span_processor:       drop_on_queue_full: false       max_queue_size: 6       batch_timeout: 2       inactive_timeout: 1       max_export_batch_size: 2

[[inputs.opentelemetry]]  ## During creating 'trace', 'span' and 'resource', many labels will be added, and these labels will eventually appear in all 'spans'  ## When you don't want too many labels to cause unnecessary traffic loss on the network, you can choose to ignore these labels  ## with setting up an regular expression list.  ## Note: ignore_attribute_keys will be effected on both trace and metrics if setted up.  # ignore_attribute_keys = ["os_*", "process_*"]  ## The acceptable http_status_ok values will be 200 or 202.       [inputs.opentelemetry.http]   enable = true   http_status_ok = 200   trace_api = "/otel/v1/traces"

{     "disable": false,     "include_resp_body": true,     "include_resp_body_expr": false,     "path": "/usr/local/apisix/logs/file.log" }

nginx_config:  http:    access_log_format: '{"remote_addr": "$remote_addr", "remote_user": "$remote_user", "time_local": "$time_local", "http_host": "$http_host", "request": "$request", "status": "$status", "body_bytes_sent": "$body_bytes_sent", "request_time": "$request_time", "http_referer": "$http_referer", "http_user_agent": "$http_user_agent", "upstream_addr": "$upstream_addr", "upstream_status": "$upstream_status", "upstream_response_time": "$upstream_response_time", "upstream_url": "$upstream_scheme://$upstream_host$upstream_uri","x-gc-trace-id": "$http_x_gc_trace_id","x-gc-span-id": "$http_x_gc_span_id","traceparent": "$http_traceparent"}'

if vaild_json(_) { # true  origin = load_json(_)  add_key(remote_addr,origin["remote_addr"])  add_key(remote_user,origin["remote_user"])  add_key(http_host,origin["http_host"])  add_key(request,origin["request"])  add_key(http_referer,origin["http_referer"])  add_key(body_bytes_sent,origin["body_bytes_sent"])  add_key(upstream_url,origin["upstream_url"])

  add_key(trace_id,origin["traceparent"])  grok(trace_id, "%{DATA}-%{DATA:trace_id}-%{DATA}") 
}

{"client_ip":"127.0.0.1","server": {"hostname":"localhost.localdomain","version":"2.14.1"},"apisix_latency":0,"upstream":"127. 0.0.1:9529","upstream_latency":1,"request": {"url":"http:\/\/127.0.0.1:80\/otel\/v1\/traces","size":836,"uri":"\/otel\/v1\/traces","que rystring":{},"headers":{"content-type":"application\/x-protobuf","user-agent":"lua-resty- http\/0.16.1 (Lua) ngx_lua\/10021","content-length":"672","traceparent":"00-749cc2e62e47bcf01fb76932c74ec1fe-d0a6101332bca613-01","host":"127.0.0.1"},"method":"POST"},"latency":0.99992752075195,"start_time":1685688047 197,"service_id":"","route_id":"462899766133524220","response": {"size":493,"status":200,"headers":{"access-control-allow-headers":"Content-Type, Content- Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X- Requested-With, X-Token, X-Datakit-UUID, X-RP, X-Precision, X-Lua","access-control-allow- credentials":"true","server":"APISIX\/2.14.1","content-length":"0","access-control-allow- origin":"*","x-datakit":"1.5.7\/2023-03-20 11:50:32","connection":"close","access-control- allow-methods":"POST, OPTIONS, GET, PUT","date":"Fri, 02 Jun 2023 06:40:47 GMT"}}}

json(_, client_ip, client_ip) json(_, upstream, upstream) json(_, request.url, url) json(_, response.status, http_status) json(_, route_id, route_id) json(_, request.headers.traceparent, trace_id) grok(_, "%{DATA}traceparent\":\"%{DATA}-%{DATA:trace_id}-%{DATA}\"")