resolved: don't attempt to send queries for DNSSEC RR types to servers not supporting them

If we already degraded the feature level below DO don't bother with sending requests for DS, DNSKEY, RRSIG, NSEC, NSEC3
or NSEC3PARAM RRs. After all, we cannot do DNSSEC validation then anyway, and we better not press a legacy server like
this with such modern concepts.

This also has the benefit that when we try to validate a response we received using DNSSEC, and we detect a limited
server support level while doing so, all further auxiliary DNSSEC queries will fail right-away.

src/libsystemd/sd-bus/bus-common-errors.h

@@ -76,6 +76,7 @@
 #define BUS_ERROR_NO_SUCH_SERVICE "org.freedesktop.resolve1.NoSuchService"
 #define BUS_ERROR_DNSSEC_FAILED "org.freedesktop.resolve1.DnssecFailed"
 #define BUS_ERROR_NO_TRUST_ANCHOR "org.freedesktop.resolve1.NoTrustAnchor"
+#define BUS_ERROR_RR_TYPE_UNSUPPORTED "org.freedesktop.resolve1.ResourceRecordTypeUnsupported"
 #define _BUS_ERROR_DNS "org.freedesktop.resolve1.DnsError."
 
 #define BUS_ERROR_NO_SUCH_TRANSFER "org.freedesktop.import1.NoSuchTransfer"

src/resolve/dns-type.c

@@ -114,6 +114,16 @@ bool dns_type_may_redirect(uint16_t type) {
                        DNS_TYPE_KEY);
 }
 
+bool dns_type_is_dnssec(uint16_t type) {
+        return IN_SET(type,
+                      DNS_TYPE_DS,
+                      DNS_TYPE_DNSKEY,
+                      DNS_TYPE_RRSIG,
+                      DNS_TYPE_NSEC,
+                      DNS_TYPE_NSEC3,
+                      DNS_TYPE_NSEC3PARAM);
+}
+
 const char *dns_class_to_string(uint16_t class) {
 
         switch (class) {

src/resolve/dns-type.h

@@ -129,6 +129,7 @@ bool dns_type_is_pseudo(uint16_t type);
 bool dns_type_is_valid_query(uint16_t type);
 bool dns_type_is_valid_rr(uint16_t type);
 bool dns_type_may_redirect(uint16_t type);
+bool dns_type_is_dnssec(uint16_t type);
 
 bool dns_class_is_pseudo(uint16_t class);
 bool dns_class_is_valid_rr(uint16_t class);

src/resolve/resolved-bus.c

@@ -67,6 +67,9 @@ static int reply_query_state(DnsQuery *q) {
         case DNS_TRANSACTION_NO_TRUST_ANCHOR:
                 return sd_bus_reply_method_errorf(q->request, BUS_ERROR_NO_TRUST_ANCHOR, "No suitable trust anchor known");
 
+        case DNS_TRANSACTION_RR_TYPE_UNSUPPORTED:
+                return sd_bus_reply_method_errorf(q->request, BUS_ERROR_RR_TYPE_UNSUPPORTED, "Server does not support requested resource record type");
+
         case DNS_TRANSACTION_RCODE_FAILURE: {
                 _cleanup_(sd_bus_error_free) sd_bus_error error = SD_BUS_ERROR_NULL;
 

src/resolve/resolved-dns-transaction.c

@@ -418,6 +418,9 @@ static int dns_transaction_open_tcp(DnsTransaction *t) {
                 if (r < 0)
                         return r;
 
+                if (t->current_features < DNS_SERVER_FEATURE_LEVEL_DO && dns_type_is_dnssec(t->key->type))
+                        return -EOPNOTSUPP;
+
                 r = dns_server_adjust_opt(t->server, t->sent, t->current_features);
                 if (r < 0)
                         return r;
@@ -696,6 +699,11 @@ void dns_transaction_process_reply(DnsTransaction *t, DnsPacket *p) {
                         dns_transaction_complete(t, DNS_TRANSACTION_NO_SERVERS);
                         return;
                 }
+                if (r == -EOPNOTSUPP) {
+                        /* Tried to ask for DNSSEC RRs, on a server that doesn't do DNSSEC  */
+                        dns_transaction_complete(t, DNS_TRANSACTION_RR_TYPE_UNSUPPORTED);
+                        return;
+                }
                 if (r < 0) {
                         /* On LLMNR, if we cannot connect to the host,
                          * we immediately give up */
@@ -832,6 +840,9 @@ static int dns_transaction_emit_udp(DnsTransaction *t) {
                 if (t->current_features < DNS_SERVER_FEATURE_LEVEL_UDP)
                         return -EAGAIN;
 
+                if (t->current_features < DNS_SERVER_FEATURE_LEVEL_DO && dns_type_is_dnssec(t->key->type))
+                        return -EOPNOTSUPP;
+
                 if (r > 0 || t->dns_udp_fd < 0) { /* Server changed, or no connection yet. */
                         int fd;
 
@@ -1277,7 +1288,13 @@ int dns_transaction_go(DnsTransaction *t) {
                 /* No servers to send this to? */
                 dns_transaction_complete(t, DNS_TRANSACTION_NO_SERVERS);
                 return 0;
-        } else if (r < 0) {
+        }
+        if (r == -EOPNOTSUPP) {
+                /* Tried to ask for DNSSEC RRs, on a server that doesn't do DNSSEC  */
+                dns_transaction_complete(t, DNS_TRANSACTION_RR_TYPE_UNSUPPORTED);
+                return 0;
+        }
+        if (r < 0) {
                 if (t->scope->protocol != DNS_PROTOCOL_DNS) {
                         dns_transaction_complete(t, DNS_TRANSACTION_RESOURCES);
                         return 0;
@@ -2764,6 +2781,7 @@ static const char* const dns_transaction_state_table[_DNS_TRANSACTION_STATE_MAX]
         [DNS_TRANSACTION_ABORTED] = "aborted",
         [DNS_TRANSACTION_DNSSEC_FAILED] = "dnssec-failed",
         [DNS_TRANSACTION_NO_TRUST_ANCHOR] = "no-trust-anchor",
+        [DNS_TRANSACTION_RR_TYPE_UNSUPPORTED] = "rr-type-unsupported",
 };
 DEFINE_STRING_TABLE_LOOKUP(dns_transaction_state, DnsTransactionState);
 

src/resolve/resolved-dns-transaction.h

@@ -39,6 +39,7 @@ enum DnsTransactionState {
         DNS_TRANSACTION_ABORTED,
         DNS_TRANSACTION_DNSSEC_FAILED,
         DNS_TRANSACTION_NO_TRUST_ANCHOR,
+        DNS_TRANSACTION_RR_TYPE_UNSUPPORTED,
         _DNS_TRANSACTION_STATE_MAX,
         _DNS_TRANSACTION_STATE_INVALID = -1
 };