Docker 健康状态与 Prometheus up 回答不同问题

from pathlib import Path
from tempfile import TemporaryDirectory
from urllib.parse import urlencode
from urllib.request import urlopen
import json,subprocess,time,uuid

BASE='python@sha256:528257d48c1da0dcecc2e725d1ae34498d60c965f1241e39cd6a85a8859bdf84'
PROM='prom/prometheus@sha256:38c3b05c3bc744ff1b0b7b4eb82196026442845e62a1e2073795565da506d7a2'
def run(args,check=True):
    result=subprocess.run(args,text=True,capture_output=True,timeout=120)
    if check:assert result.returncode==0,result.stderr
    return result
def docker(*args,check=True):return run(['docker',*args],check)
def inspect(identity):return json.loads(docker('inspect',identity).stdout)[0]

name='xbcatm-prom-health-'+uuid.uuid4().hex[:10];tag=name+':prometheus'
with TemporaryDirectory(prefix=name+'-') as directory:
    root=Path(directory);file=root/'compose.yaml'
    def compose(*args,check=True):return run(['docker','compose','-p',name,'-f',str(file),*args],check)
    def wait(predicate):
        deadline=time.monotonic()+25
        while time.monotonic()<deadline:
            try:
                if predicate():return
            except (OSError,ValueError):pass
            time.sleep(.2)
        raise AssertionError('observation did not become true')
    server='''from http.server import BaseHTTPRequestHandler,HTTPServer
from pathlib import Path
Path('/ready').touch()
class H(BaseHTTPRequestHandler):
 def do_GET(self):
  ready=Path('/ready').exists()
  code=(503 if Path('/broken-metrics').exists() else 200) if self.path=='/metrics' else (200 if ready else 503)
  body=('# TYPE app_ready gauge\\napp_ready '+str(int(ready))+'\\n').encode()
  self.send_response(code);self.send_header('Content-Type','text/plain; version=0.0.4');self.end_headers();self.wfile.write(body)
 def log_message(self,*args):pass
HTTPServer(('0.0.0.0',8080),H).serve_forever()
'''
    (root/'prometheus.yml').write_text('global:\n  scrape_interval: 1s\n  scrape_timeout: 500ms\nscrape_configs:\n  - job_name: app\n    static_configs:\n      - targets: [app:8080]\n')
    (root/'Dockerfile').write_text(f'FROM {PROM}\nCOPY prometheus.yml /etc/prometheus/prometheus.yml\n')
    health=['CMD','python','-c','import urllib.request;urllib.request.urlopen("http://127.0.0.1:8080/health",timeout=.5).read()']
    file.write_text(json.dumps({'services':{'app':{'image':BASE,'command':['python','-c',server],'restart':'always','healthcheck':{'test':health,'interval':'1s','timeout':'1s','retries':2,'start_period':'3s'}},'prometheus':{'image':tag,'ports':[{'target':9090,'published':'0','host_ip':'127.0.0.1'}],'volumes':['metrics:/prometheus']}},'volumes':{'metrics':{}}},indent=2))
    try:
        docker('build','--network','none','-t',tag,str(root));compose('up','-d')
        app=compose('ps','-q','app').stdout.strip();prometheus=compose('ps','-q','prometheus').stdout.strip()
        port=inspect(prometheus)['NetworkSettings']['Ports']['9090/tcp'][0];assert port['HostIp']=='127.0.0.1'
        endpoint='http://127.0.0.1:'+port['HostPort']
        def value(expression):
            with urlopen(endpoint+'/api/v1/query?'+urlencode({'query':expression}),timeout=2) as response:data=json.load(response)
            result=data['data']['result'];return float(result[0]['value'][1]) if result else None
        wait(lambda:value('up{job="app"}')==1 and value('app_ready')==1 and inspect(app)['State']['Health']['Status']=='healthy')
        time.sleep(11)
        original=inspect(app)
        docker('exec',app,'python','-c','from pathlib import Path;Path("/ready").unlink()')
        wait(lambda:value('up{job="app"}')==1 and value('app_ready')==0 and inspect(app)['State']['Health']['Status']=='unhealthy')
        current=inspect(app);assert current['RestartCount']==0 and current['State']['StartedAt']==original['State']['StartedAt']
        print('Docker unhealthy and app_ready=0 coexist with Prometheus up=1 because the metrics endpoint is still scrapeable')
        docker('exec',app,'python','-c','from pathlib import Path;Path("/broken-metrics").touch()')
        wait(lambda:value('up{job="app"}')==0)
        assert inspect(app)['State']['Running'] and inspect(app)['RestartCount']==0
        print('returning HTTP 503 from /metrics changes up to 0 without stopping or restarting the application')
    finally:
        compose('down','--volumes','--remove-orphans',check=False);docker('image','rm',tag,check=False)