nginx 被动上游健康判断利用真实请求结果,决定某个节点是否暂时停止接收流量。max_fails 设置失败尝试阈值,fail_timeout 同时规定计数区间和不可用时长。这些配置不会安排周期性的应用探测。主动检查需要被支持的检查能力与额外配置,不能把真实请求结果策略直接称作一个已经运行的探活定时器。
坏掉的样子 不是每个看起来像错误的 HTTP 响应,都算一次失败上游尝试。代理重试配置参与定义这种分类。在本次设置中,只有列入 http_503,HTTP 503 才参与相应被动失败处理。如果认为所有 5xx 都会自动导致健康失败,就可能期待节点已被摘除,而 nginx 仍依据真实规则继续选择它。必须先核对什么结果被配置认定为失败。
另一个错误是把超时窗口结束当成恢复证明。不可用时段结束后,节点可以重新获得真实流量资格,未必先通过独立健康探测。如果后端仍坏着,这次请求仍会失败。least_conn 依据活动连接改变选择,并不提供渐进恢复权重的升温过程,也不能代替受支持的 slow-start 机制。资格恢复、健康证明和逐渐加量需要分别理解。
重试也有边界。响应数据已经发送给客户端后,更换节点不能透明修复中途错误;重放有副作用的操作还可能产生重复。下面只使用 GET,并在成功正文发送之前产生失败,且明确限制尝试次数。它没有为所有应用方法证明安全重试语义,不能把这个成功切换案例直接用于任何写操作。
快速验证 将脚本保存为 check-passive-health.py,准备 Python、Docker 和固定镜像后运行。记录服务器为单 worker 的 nginx 1.28.0。可丢弃容器里的两个 HTTP 端点报告自身身份,并能受控返回 503。另一个控制监听提供计数,查询它不会通过被均衡组产生业务请求,便于区分观察动作与真实探测流量:
from contextlib import contextmanager from pathlib import Path from tempfile import TemporaryDirectory import http.client,json,socket,subprocess,time,uuid NGINX='nginx@sha256:552e7481ca93ffccd046aa658dbbed22caefbc09c66fa7cd247cbb90b8a5c609' PYTHON='python@sha256:528257d48c1da0dcecc2e725d1ae34498d60c965f1241e39cd6a85a8859bdf84' def docker(*args,check=True): result=subprocess.run(['docker',*args],capture_output=True,text=True,timeout=120) if check:assert result.returncode==0,result.stderr return result def wait(predicate): deadline=time.monotonic()+15 while time.monotonic()<deadline: try: if predicate():return except OSError:pass time.sleep(.1) raise AssertionError('fixture did not become ready') @contextmanager def lab(configuration,files=None,backend=None,extra_ports=()): name='xbcatm-nginx-'+uuid.uuid4().hex[:10];tags=[name+':proxy',name+':backend'];containers=[name+'-proxy',name+'-backend'] try: with TemporaryDirectory(prefix=name+'-') as directory: root=Path(directory);(root/'files').mkdir();(root/'nginx.conf').write_text(configuration) for path,content in (files or {}).items(): target=root/'files'/path;target.parent.mkdir(parents=True,exist_ok=True) target.write_bytes(content if isinstance(content,bytes) else content.encode()) docker('network','create',name) if backend is not None: (root/'backend.py').write_text(backend) (root/'backend.Dockerfile').write_text(f'FROM {PYTHON}\nCOPY backend.py /backend.py\nCMD ["python", "/backend.py"]\n') docker('build','--network','none','-f',str(root/'backend.Dockerfile'),'-t',tags[1],str(root)) docker('run','-d','--name',containers[1],'--network',name,'--network-alias','backend',tags[1]) wait(lambda:docker('exec',containers[1],'python','-c','import socket;s=socket.create_connection(("127.0.0.1",8081),timeout=.5);s.close()',check=False).returncode==0) (root/'Dockerfile').write_text(f'FROM {NGINX}\nCOPY nginx.conf /etc/nginx/nginx.conf\nCOPY files /srv\n') docker('build','--network','none','-t',tags[0],str(root)) docker('run','--rm','--network',name,tags[0],'nginx','-t') published=[arg for value in (8080,*extra_ports) for arg in ('-p','127.0.0.1::'+str(value))] docker('run','-d','--name',containers[0],'--network',name,*published,tags[0]) details=json.loads(docker('inspect',containers[0]).stdout)[0] binding=details['NetworkSettings']['Ports']['8080/tcp'][0];assert binding['HostIp']=='127.0.0.1';port=int(binding['HostPort']) def ready(): connection=socket.create_connection(('127.0.0.1',port),timeout=.5);connection.close();return True wait(ready) def request(path,method='GET',headers=None,body=None): connection=http.client.HTTPConnection('127.0.0.1',port,timeout=5) try: connection.request(method,path,body=body,headers=headers or {}) response=connection.getresponse();return response.status,{k.lower():v for k,v in response.getheaders()},response.read() finally:connection.close() request.port=port;request.network=name request.ports={int(key.split('/')[0]):int(value[0]['HostPort']) for key,value in details['NetworkSettings']['Ports'].items() if value} yield containers[0],request finally: docker('rm','-f',*containers,check=False);docker('image','rm',*tags,check=False);docker('network','rm',name,check=False) def config(locations,extra=''): return 'worker_processes 1;\nerror_log /tmp/error.log notice;\nevents {}\nhttp {\naccess_log off;\n'+extra+'\nserver { listen 8080; root /srv;\n'+locations+'\n}\n}\n' backend=r''' from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer from urllib.parse import urlsplit,parse_qs import itertools,json,threading lock=threading.Lock();ids=itertools.count(1);counts={'A':0,'B':0};active={'A':0,'B':0};faults={'A':200,'B':200};gates={} class Handler(BaseHTTPRequestHandler): protocol_version='HTTP/1.1' def setup(self):super().setup();self.ident=next(ids) def log_message(self,*args):pass def do_GET(self): parsed=urlsplit(self.path);label=self.server.label;code=200 if label=='control': query=parse_qs(parsed.query) with lock: if parsed.path=='/fail':faults[query['peer'][0]]=int(query['status'][0]) if parsed.path=='/release':gates[query['path'][0]].set() data={'counts':dict(counts),'active':dict(active),'faults':dict(faults),'held':list(gates)} else: with lock: counts[label]+=1;active[label]+=1;code=faults[label] if parsed.path.startswith('/hold/'):gates[parsed.path]=threading.Event() if parsed.path.startswith('/hold/'):assert gates[parsed.path].wait(20) data={'peer':label,'path':self.path,'connection':self.ident} body=json.dumps(data,sort_keys=True).encode();self.send_response(code) self.send_header('Content-Type','application/json');self.send_header('Content-Length',str(len(body))) self.end_headers();self.wfile.write(body) if label!='control': with lock:active[label]-=1 for port,label in [(8081,'A'),(8082,'B'),(8083,'control')]: server=ThreadingHTTPServer(('0.0.0.0',port),Handler);server.label=label threading.Thread(target=server.serve_forever,daemon=True).start() threading.Event().wait() ''' extra=''' upstream passive { server backend:8081 max_fails=1 fail_timeout=2s; server backend:8082; } upstream plain { server backend:8081 max_fails=1 fail_timeout=30s; server backend:8082; } upstream single { server backend:8081 max_fails=1 fail_timeout=30s; } add_header X-Upstream $upstream_addr always; add_header X-Attempts $upstream_status always; ''' locations=''' location /passive { proxy_next_upstream error timeout http_503; proxy_next_upstream_tries 2; proxy_pass http://passive; } location /plain { proxy_next_upstream error timeout; proxy_pass http://plain; } location /single { proxy_next_upstream error timeout http_503; proxy_pass http://single; } location /control/ { proxy_pass http://backend:8083/; } ''' active_config=config('location / { proxy_pass http://peers; health_check; }', 'upstream peers { zone peers 64k; server backend:8081; server backend:8082; }') with lab(config(locations,extra),{'active.conf':active_config},backend) as (proxy,request): def control(path): status,_,body=request('/control/'+path);assert status==200;return json.loads(body) def call(path): status,h,b=request(path);return {'status':status,'attempts':h['x-attempts'],'upstream':h['x-upstream'],'peer':json.loads(b)['peer']} control('fail?peer=A&status=503');results={} plain=[call('/plain') for _ in range(6)] assert [r['status'] for r in plain].count(503)==3 and all(',' not in r['attempts'] for r in plain) results['unlisted_http_503']=plain for _ in range(2): first=call('/passive') if first['attempts']=='503, 200':break assert first['status']==200 and first['attempts']=='503, 200' and first['peer']=='B',first results['failed_attempt_then_success']=first after_failure=control('state')['counts']['A'] excluded=[call('/passive') for _ in range(4)] assert all(r['peer']=='B' and r['attempts']=='200' for r in excluded) assert control('state')['counts']['A']==after_failure time.sleep(.4);assert control('state')['counts']['A']==after_failure results['during_exclusion']=excluded;results['background_probe_count']=0 control('fail?peer=A&status=200');deadline=time.monotonic()+6;recovered=None while time.monotonic()<deadline: observed=call('/passive') if observed['peer']=='A':recovered=observed;break time.sleep(.1) assert recovered and recovered['status']==200;results['live_request_after_window']=recovered control('fail?peer=A&status=503');before=control('state')['counts']['A'] single=[call('/single') for _ in range(2)] assert all(r['status']==503 and r['attempts']=='503' for r in single) assert control('state')['counts']['A']==before+2;results['single_peer_still_attempted']=single probe=docker('exec',proxy,'nginx','-t','-c','/srv/active.conf',check=False) assert probe.returncode!=0 and 'unknown directive "health_check"' in probe.stderr results['active_health_check_directive_supported']=False print(json.dumps(results,sort_keys=True)) 首先让 A 返回真实 HTTP 503,B 返回 200。一个对照组只对连接错误和超时重试。六次请求仍各选择 A、B 三次,没有多次尝试的响应。A 的 max_fails=1 不会因为这个未列入的 HTTP 结果把它移除。实验检查状态与所选上游地址,不只查看最终正文,从而避免把应用返回值与节点资格变化混淆。
...