Traceroute 发送 TTL 逐步增大的探测包,观察数据包在哪里到期,以及哪些系统报告了到期。IPv4 路由器转发时递减 TTL;到期后,通常由 ICMP Time Exceeded 响应标识一个作出回复的跳点。传统 UDP 跟踪在目标报告端口未使用时结束。输出是一系列关于探测和回复的观察,不保证列出全部转发设备,也不直接等同于应用经过的唯一稳定路径。

坏掉的样子

星号表示某个探测没有在等待时间内获得预期响应。路由器可能仍正常转发普通流量,却过滤、抑制或限制自己生成的 ICMP 回复。回复的返回路径也可能出问题。因此,中间某跳沉默,本身不能证明应用数据包在那里停止;后续跳点仍然回复,甚至会直接反驳这种判断。需要区分转发动作和产生诊断响应这两种行为。

探测协议同样重要。传统高端口 UDP 流量可能遇到与允许服务端口上的 TCP 不同的策略。TCP traceroute 可以让探测更接近被允许的传输方式,但仍没有执行完整应用交互。收到 SYN 响应或重置,可以帮助判断目的地可达,却不能证明 HTTP、认证或某项业务操作成功。协议层面的终点响应必须与应用结果分别解释。

每跳时间是从观察者出发的往返时间,包含响应生成和返回路径。把相邻数值相减,不能可靠测量两台路由器之间的单向链路延迟。负载均衡、返回路径不对称、调度与 ICMP 处理都会影响观察。比较跟踪时,应保留探测方式和测量条件,不能把一次打印序列当作永久拓扑,也不能把后续更短的往返值解释成负延迟。

快速验证

将脚本保存为 check-traceroute.py,准备 Docker 后运行 python3 check-traceroute.py。一次性环境使用 traceroute 2.1.2、nftables 1.0.6 与 Python 3.11.16。NET_ADMIN 和 SYS_ADMIN 支持嵌套命名空间及临时 proc 挂载。客户端、路由器和目标通过新建 veth 相连,全部选路与过滤变更都留在实验内部,不触碰宿主正在使用的网络配置:

from pathlib import Path
from tempfile import TemporaryDirectory
import json,subprocess,uuid
recipe='FROM python@sha256:528257d48c1da0dcecc2e725d1ae34498d60c965f1241e39cd6a85a8859bdf84\nRUN apt-get update && apt-get install -y --no-install-recommends iproute2=6.1.0-3 iputils-ping=3:20221126-1+deb12u1 nftables=1.0.6-2+deb12u2 tcpdump=4.99.3-1 traceroute=1:2.1.2-1 procps=2:4.0.2-3 && rm -rf /var/lib/apt/lists/*\nCMD ["python3"]\n'
program=r"""
from contextlib import ExitStack,contextmanager
from pathlib import Path
from tempfile import TemporaryDirectory
import json,os,select,signal,subprocess,time

def run(*args,check=True,input=None,timeout=8):
    result=subprocess.run(args,input=input,capture_output=True,text=True,timeout=timeout)
    if check:assert result.returncode==0,(args,result.returncode,result.stdout,result.stderr)
    return result
def ns(name,*args,**kwargs):return run('ip','netns','exec',name,*args,**kwargs)
def ip(name,*args,**kwargs):return ns(name,'ip',*args,**kwargs)
def stop(process):
    if process.poll() is None:
        process.terminate()
        try:process.wait(timeout=2)
        except subprocess.TimeoutExpired:process.kill();process.wait(timeout=2)
def line(process,timeout=3):
    assert select.select([process.stdout],[],[],timeout)[0],'child response deadline exceeded'
    value=process.stdout.readline();assert value,'child exited without a response'
    return value
def spawn(stack,command):
    process=subprocess.Popen(command,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
    stack.callback(stop,process);return process
@contextmanager
def lab(names):
    with TemporaryDirectory(prefix='net-path-') as directory,ExitStack() as stack:
        root=Path(directory);proc=root/'proc';proc.mkdir();run('mount','-t','proc','proc',str(proc))
        stack.callback(run,'umount',str(proc))
        for name in names:
            run('ip','netns','add',name);stack.callback(run,'ip','netns','delete',name,check=False)
            ip(name,'link','set','lo','up')
        def setting(name,key,value):
            assert name in names
            path=proc/'sys'/key.replace('.','/')
            return ns(name,'python3','-c',f'from pathlib import Path; p=Path({str(path)!r}); p.write_text({str(value)!r}); print(p.read_text().strip())').stdout.strip()
        yield root,stack,setting
def wire(left,left_if,right,right_if,left_addresses,right_addresses):
    run('ip','link','add',left_if,'type','veth','peer','name',right_if)
    for name,interface,addresses in [(left,left_if,left_addresses),(right,right_if,right_addresses)]:
        run('ip','link','set',interface,'netns',name)
        for address in addresses:ip(name,'address','add',address,'dev',interface,*(['nodad'] if ':' in address else []))
        ip(name,'link','set',interface,'up')
def nft(name,script):return ns(name,'nft','-f','-',input=script)
def counters(name):
    data=json.loads(ns(name,'nft','-j','list','ruleset').stdout);found={}
    for entry in data['nftables']:
        rule=entry.get('rule',{})
        if 'comment' in rule:
            for expression in rule['expr']:
                if 'counter' in expression:found[rule['comment']]=expression['counter']['packets']
    return found

def server(stack,name,address,port=18080,label='server'):
    code=r'''
from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
import json,socket,sys
address,port,label=sys.argv[1],int(sys.argv[2]),sys.argv[3]
class Handler(BaseHTTPRequestHandler):
    def do_GET(self):
        body=json.dumps({'server':label,'peer':self.client_address,'path':self.path}).encode()
        self.send_response(200);self.send_header('Content-Length',str(len(body)));self.end_headers();self.wfile.write(body)
    def log_message(self,*args):pass
class Server(ThreadingHTTPServer):
    address_family=socket.AF_INET6 if ':' in address else socket.AF_INET
    def server_bind(self):
        if self.address_family==socket.AF_INET6:self.socket.setsockopt(socket.IPPROTO_IPV6,socket.IPV6_V6ONLY,1)
        super().server_bind()
service=Server((address,port),Handler);print('ready',flush=True);service.serve_forever()
'''
    process=spawn(stack,['ip','netns','exec',name,'python3','-u','-c',code,address,str(port),label])
    assert line(process).strip()=='ready';return process
def request(name,address,port=18080):
    code=r'''
import http.client,json,sys
client=http.client.HTTPConnection(sys.argv[1],int(sys.argv[2]),timeout=.7)
try:
    client.request('GET','/lab-visible-message');response=client.getresponse();body=json.loads(response.read())
    assert response.status==200;print(json.dumps({'ok':True,'status':response.status,'body':body}))
except OSError as error:print(json.dumps({'ok':False,'type':type(error).__name__,'errno':error.errno}))
finally:client.close()
'''
    return json.loads(ns(name,'python3','-c',code,address,str(port)).stdout)

def routed(setting):
    wire('client','c0','router','r0',['10.201.1.2/30'],['10.201.1.1/30'])
    wire('router','r1','target','t0',['10.201.2.1/30'],['10.201.2.2/30'])
    setting('router','net.ipv4.ip_forward',1)
    ip('client','route','add','default','via','10.201.1.1')
    ip('target','route','add','default','via','10.201.2.1')
with lab(['client','router','target']) as (root,stack,setting):
    routed(setting);server(stack,'target','10.201.2.2')
    def trace(*options):return ns('client','traceroute','-n','-q','1','-w','0.3','-m','4',*options,'10.201.2.2')
    initial=trace();assert '10.201.1.1' in initial.stdout and '10.201.2.2' in initial.stdout.splitlines()[-1]
    nft('router','''table ip quiet {
 chain output { type filter hook output priority 0; policy accept;
  icmp type time-exceeded counter drop comment "hidden_hop"
 }
}
''')
    silent=trace();rows=silent.stdout.splitlines();assert '*' in rows[1] and '10.201.2.2' in rows[2]
    still_works=request('client','10.201.2.2');assert still_works['ok'] and counters('router')['hidden_hop']>0
    nft('router','delete table ip quiet')
    nft('target','''table ip udp_filter {
 chain input { type filter hook input priority 0; policy accept;
  udp dport 33434-33500 counter drop comment "udp_probe_drop"
 }
}
''')
    udp_blocked=trace();assert '10.201.1.1' in udp_blocked.stdout and all('10.201.2.2' not in row for row in udp_blocked.stdout.splitlines()[1:])
    tcp=trace('-T','-p','18080');assert '10.201.2.2' in tcp.stdout.splitlines()[-1]
    assert counters('target')['udp_probe_drop']>0 and request('client','10.201.2.2')['ok']
    print(json.dumps({'ordinary_udp_trace':initial.stdout,'router_time_exceeded_filtered':silent.stdout,
        'http_works_with_silent_hop':still_works,'destination_udp_filtered':udp_blocked.stdout,
        'udp_trace_exit_despite_missing_destination':udp_blocked.returncode,'tcp_trace_to_service':tcp.stdout,
        'target_filter_counters':counters('target')}))
"""
name='xbcatm-netpath-'+uuid.uuid4().hex[:10]
def docker(*args,**kwargs):return subprocess.run(['docker',*args],capture_output=True,text=True,timeout=300,**kwargs)
try:
    with TemporaryDirectory(prefix=name+'-') as directory:
        root=Path(directory);(root/'Dockerfile').write_text(recipe)
        build=docker('build','-t',name,str(root));assert build.returncode==0,build.stderr
        result=docker('run','--rm','-i','--name',name,'--network','none','--cap-add','NET_ADMIN','--cap-add','SYS_ADMIN',
            '--memory','256m','--pids-limit','128',name,'python3','-',input=program)
        assert result.returncode==0,result.stderr
        print(json.dumps(json.loads(result.stdout),sort_keys=True))
finally:
    docker('rm','-f',name);docker('image','rm',name)

目标在 TCP 端口 18080 监听真实 HTTP 请求。正常转发且没有新增过滤时,使用数字地址的 UDP 跟踪在第一跳显示路由器 10.201.1.1,第二跳显示目标 10.201.2.2。脚本每跳只发一个探测,使用较短回复等待和四跳上限。这些有界参数适合本地小拓扑,并不是推荐给所有广域网络的统一超时值。

接着,路由器在输出链中仅丢弃自己生成的 ICMP Time Exceeded 消息。第一跳变为星号,目标仍在第二跳回复。同时,实际 HTTP 请求成功,路由器过滤计数器增长。这个对照把诊断反馈丢失与转发丢失区分开:即使缺少该跳响应,应用流量仍然能够通过路由器,不能直接把星号所在位置标为故障断点。

删除路由器过滤规则后,在目标丢弃默认跟踪方式使用的高端口 UDP。新的输出显示路由器,后面均为星号,没有目标响应。然而在本次测试的 traceroute 版本中,进程退出状态仍为零。因此,命令成功退出不能作为“已经观察到预期目的地”的断言。自动化必须检查实际输出含义,而不是把运行结束与路径验证混为一谈。

保留目标 UDP 过滤规则,脚本改用 -T -p 18080 发送 TCP 探测。此时跟踪到达目标,独立 HTTP 请求也成功。UDP 丢弃计数证明先前探测经过预期规则。改变探测方式即可解释两种输出,无需改变真实应用的监听端口或服务行为。HTTP 成功提供应用证据,TCP 跟踪提供探测响应证据,两者相互补充但不互相替代。

用跟踪缩小待验证的原因

尽可能从应用实际使用的来源、目的地、地址族和传输协议开始,运行应用探测并在相关位置重复比较。从笔记本发起的结果,可能不同于容器命名空间或另一张具有不同路由和策略的网络。脚本用隔离命名空间明确来源与报文路径,避免依赖宿主偶然存在的配置;在生产排查中,也要保存相同维度的上下文才能比较。

遇到沉默跳点时,先检查后续响应和端到端行为,再将故障归属到该路由器。只有一种跟踪方式失败时,应检查针对该方式的策略,不能假定所有协议命运相同。自动化应解析相关目的地结果,并结合有意义的服务检查。保存原始跟踪文本与时间条件,让之后的审阅者能够分清缺少证据和已经证明失败,避免单个星号驱动错误修复。

延伸阅读

来源