zonemaster-tools/zmtools/resolvers.py

"""DNS propagation and global resolver testing.

Propagation asks the same question of many public resolvers in many countries
and compares the answers. Anycast services answer from whichever node is
nearest to gringo (Europe), so for them we also ask Google with an EDNS
Client Subnet from each region: that is what geo-steered CDNs actually key
their answers on.

Resolver testing compares *how* resolvers answer — UDP, TCP and DoH latency,
DNSSEC validation (AD bit), and rcode — including a resolver the user names.
"""

from concurrent.futures import ThreadPoolExecutor

import dns.flags
import dns.rdatatype

from . import dnsutil

# (id, label, country code or ANY for anycast, city, ip). The UI localises
# country names with Intl.DisplayNames.
PROPAGATION = [
    ('cloudflare', 'Cloudflare', 'ANY', '', '1.1.1.1'),
    ('google', 'Google Public DNS', 'ANY', '', '8.8.8.8'),
    ('quad9', 'Quad9', 'ANY', '', '9.9.9.10'),
    ('opendns', 'OpenDNS (Cisco)', 'ANY', '', '208.67.222.222'),
    ('adguard', 'AdGuard (unfiltered)', 'ANY', '', '94.140.14.140'),
    ('controld', 'Control D', 'ANY', '', '76.76.2.0'),
    ('dnssb', 'DNS.SB', 'ANY', '', '185.222.222.222'),
    ('level3', 'Level 3 / Lumen', 'US', '', '4.2.2.2'),
    ('he', 'Hurricane Electric', 'US', 'Fremont', '74.82.42.42'),
    ('comodo', 'Comodo Secure DNS', 'US', '', '8.26.56.26'),
    ('cira', 'CIRA Canadian Shield', 'CA', '', '149.112.121.10'),
    ('dnswatch', 'DNS.WATCH', 'DE', '', '84.200.69.80'),
    ('cznic', 'CZ.NIC ODVR', 'CZ', '', '193.17.47.1'),
    ('yandex', 'Yandex DNS', 'RU', '', '77.88.8.8'),
    ('alidns', 'AliDNS', 'CN', '', '223.5.5.5'),
    ('dnspod', 'DNSPod (Tencent)', 'CN', '', '119.29.29.29'),
    ('114dns', '114DNS', 'CN', '', '114.114.114.114'),
    ('kt', 'KT', 'KR', '', '168.126.63.1'),
]

# Google honours ECS, so these approximate a user in each region.
ECS_REGIONS = [
    ('ecs-es', 'Madrid', 'ES', '88.0.0.0'),
    ('ecs-de', 'Frankfurt', 'DE', '91.0.0.0'),
    ('ecs-gb', 'London', 'GB', '81.2.69.0'),
    ('ecs-us-e', 'New York', 'US', '24.104.0.0'),
    ('ecs-us-w', 'California', 'US', '64.124.0.0'),
    ('ecs-br', 'São Paulo', 'BR', '177.0.0.0'),
    ('ecs-in', 'Mumbai', 'IN', '49.32.0.0'),
    ('ecs-sg', 'Singapore', 'SG', '116.12.0.0'),
    ('ecs-jp', 'Tokyo', 'JP', '126.0.0.0'),
    ('ecs-au', 'Sydney', 'AU', '1.120.0.0'),
    ('ecs-za', 'Johannesburg', 'ZA', '41.0.0.0'),
]

# (id, label, udp ip, DoH url)
TEST_RESOLVERS = [
    ('cloudflare', 'Cloudflare', '1.1.1.1', 'https://cloudflare-dns.com/dns-query'),
    ('google', 'Google', '8.8.8.8', 'https://dns.google/dns-query'),
    ('quad9', 'Quad9', '9.9.9.9', 'https://dns.quad9.net/dns-query'),
    ('opendns', 'OpenDNS', '208.67.222.222', 'https://doh.opendns.com/dns-query'),
    ('adguard', 'AdGuard DNS', '94.140.14.14', 'https://dns.adguard-dns.com/dns-query'),
    ('controld', 'Control D', '76.76.2.0', 'https://freedns.controld.com/p0'),
    ('mullvad', 'Mullvad', None, 'https://dns.mullvad.net/dns-query'),
    ('dnssb', 'DNS.SB', '185.222.222.222', 'https://doh.dns.sb/dns-query'),
    ('cira', 'CIRA', '149.112.121.10', 'https://private.canadianshield.cira.ca/dns-query'),
    ('alidns', 'AliDNS', '223.5.5.5', 'https://dns.alidns.com/dns-query'),
    ('cznic', 'CZ.NIC', '193.17.47.1', 'https://odvr.nic.cz/doh'),
]

TYPES = ('A', 'AAAA', 'CNAME', 'MX', 'NS', 'TXT', 'SOA', 'CAA', 'SRV', 'PTR', 'DS', 'DNSKEY', 'HTTPS')

_pool = ThreadPoolExecutor(max_workers=48, thread_name_prefix='res')


def _type(raw):
    t = (raw or 'A').upper()
    if t not in TYPES:
        raise dnsutil.BadInput('unsupported record type')
    return t


def _summ(resp, rdtype):
    vals = sorted(a['value'] for a in dnsutil.answers(resp, dns.rdatatype.from_text(rdtype))
                  if a['type'] == rdtype)
    ttls = [a['ttl'] for a in dnsutil.answers(resp) if a['type'] == rdtype]
    return vals, (min(ttls) if ttls else None)


def propagation(domain, rdtype):
    rdtype = _type(rdtype)
    if rdtype == 'PTR':
        domain = dnsutil.reverse_name(domain) if dnsutil.is_public_ip(domain) else domain

    def one(spec):
        rid, label, cc, where, ip = spec
        r, ms, err = dnsutil.query(ip, domain, rdtype)
        vals, ttl = _summ(r, rdtype)
        return {'id': rid, 'label': label, 'country': cc, 'location': where, 'ip': ip,
                'kind': 'resolver', 'rcode': dnsutil.rcode_text(r), 'error': err,
                'values': vals, 'ttl': ttl, 'ms': ms}

    def ecs(spec):
        rid, where, cc, net = spec
        r, ms, err = dnsutil.query('8.8.8.8', domain, rdtype, ecs=(net, 24))
        vals, ttl = _summ(r, rdtype)
        return {'id': rid, 'label': 'Google + ECS', 'country': cc, 'location': where,
                'ip': f'{net}/24', 'kind': 'ecs', 'rcode': dnsutil.rcode_text(r),
                'error': err, 'values': vals, 'ttl': ttl, 'ms': ms}

    futs = [_pool.submit(one, s) for s in PROPAGATION] + [_pool.submit(ecs, s) for s in ECS_REGIONS]
    # The authoritative answer is the reference everyone should converge to.
    auth = authoritative_answer(domain, rdtype)
    rows = [f.result() for f in futs]

    ref = auth['values'] if auth and auth.get('values') else None
    if ref is None:
        counts = {}
        for r in rows:
            if r['values']:
                k = tuple(r['values'])
                counts[k] = counts.get(k, 0) + 1
        ref = list(max(counts, key=counts.get)) if counts else []
    for r in rows:
        if r['error']:
            r['state'] = 'error'
        elif r['values'] == ref:
            r['state'] = 'match'
        elif not r['values']:
            r['state'] = 'empty'
        elif set(r['values']) & set(ref):
            r['state'] = 'partial'
        else:
            r['state'] = 'differs'
    answered = [r for r in rows if not r['error']]
    matching = sum(1 for r in answered if r['state'] == 'match')
    return {'domain': domain, 'type': rdtype, 'authoritative': auth, 'reference': ref,
            'results': rows, 'summary': {
                'total': len(rows), 'answered': len(answered), 'matching': matching,
                'percent': round(100 * matching / len(answered)) if answered else 0,
                'distinct': len({tuple(r['values']) for r in answered})}}


def authoritative_answer(domain, rdtype):
    """Ask one of the zone's own nameservers (found by walking from the root)."""
    steps = dnsutil.delegation_chain(domain, rdtype)
    if not steps:
        return None
    last = steps[-1]
    if last.get('authoritative'):
        vals = sorted(a['value'] for a in last.get('answer', []) if a['type'] == rdtype)
        ttls = [a['ttl'] for a in last.get('answer', []) if a['type'] == rdtype]
        return {'server': last['server'], 'ip': last['server_ip'], 'rcode': last['rcode'],
                'values': vals, 'ttl': max(ttls) if ttls else None}
    return {'error': last.get('error', 'unknown')}


def resolver_test(domain, rdtype, custom=None):
    rdtype = _type(rdtype)
    specs = list(TEST_RESOLVERS)
    if custom:
        specs.insert(0, ('custom', custom, custom, None))

    def udp(ip):
        r, ms, err = dnsutil.query(ip, domain, rdtype, dnssec=True)
        return r, ms, err

    def tcp(ip):
        return dnsutil.query(ip, domain, rdtype, tcp=True, dnssec=True)

    jobs = {}
    for rid, label, ip, doh in specs:
        if ip:
            jobs[(rid, 'udp')] = _pool.submit(udp, ip)
            jobs[(rid, 'tcp')] = _pool.submit(tcp, ip)
        if doh:
            jobs[(rid, 'doh')] = _pool.submit(dnsutil.query_doh, doh, domain, rdtype)

    out = []
    for rid, label, ip, doh in specs:
        row = {'id': rid, 'label': label, 'ip': ip, 'doh': doh, 'protocols': {}}
        for proto in ('udp', 'tcp', 'doh'):
            f = jobs.get((rid, proto))
            if f is None:
                continue
            r, ms, err = f.result()
            vals, ttl = _summ(r, rdtype)
            row['protocols'][proto] = {
                'ms': ms, 'error': err, 'rcode': dnsutil.rcode_text(r), 'values': vals,
                'ttl': ttl, 'ad': bool(r is not None and r.flags & dns.flags.AD)}
        ok = [p for p in row['protocols'].values() if not p['error']]
        row['validates'] = any(p['ad'] for p in ok)
        row['best_ms'] = min((p['ms'] for p in ok), default=None)
        out.append(row)
    ranked = sorted((r for r in out if r['best_ms'] is not None), key=lambda r: r['best_ms'])
    return {'domain': domain, 'type': rdtype, 'results': out,
            'fastest': ranked[0]['id'] if ranked else None}