dnsdist: Implement OTLP RemoteLogger

pdns/dnsdistdist/dnsdist-configuration-yaml.cc

@@ -46,6 +46,7 @@
 #include "dnsdist-web.hh"
 #include "dnsdist-xsk.hh"
 #include "fstrm_logger.hh"
+#include "otlp_logger.hh"
 #include "iputils.hh"
 #include "mmdb.hh"
 #include "remote_logger.hh"
@@ -1066,16 +1067,23 @@ static void handleLoggingConfiguration(const Context& context, const dnsdist::ru
     for (const auto& internal_trace_config : settings.open_telemetry_tracing.internal_tracing) {
       dnsdist::configuration::updateRuntimeConfiguration([context, internal_trace_config](dnsdist::configuration::RuntimeConfiguration& config) {
         if (internal_trace_config.kind == "maintenance") {
+          config.d_opentelemetryMaintenanceInterval = internal_trace_config.sample_rate == 0 ? 60 : internal_trace_config.sample_rate;
           std::vector<std::shared_ptr<RemoteLoggerInterface>> loggers;
           for (const auto& logger_name : internal_trace_config.remote_loggers) {
             auto logger = dnsdist::configuration::yaml::getRegisteredTypeByName<RemoteLoggerInterface>(std::string(logger_name));
-            if (!logger && !(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
-              throw std::runtime_error("Unable to find the protobuf logger named '" + std::string(logger_name) + "'");
+            if (!logger) {
+              if (!(dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode)) {
+                throw std::runtime_error("Unable to find the remote logger named '" + std::string(logger_name) + "'");
+              }
+              continue;
+            }
+            RemoteLoggerInterface& remoteLoggerRef = *logger;
+            if (typeid(remoteLoggerRef) != typeid(RemoteLogger) && typeid(remoteLoggerRef) != typeid(OTLPLogger)) {
+              throw std::runtime_error("The remote logger '" + std::string(logger_name) + "' is not a Protobuf or OTLP logger and can not be used for maintenance traces");
             }
             loggers.push_back(std::move(logger));
           }
           config.d_maintenanceRemoteLoggers = std::move(loggers);
-          config.d_opentelemetryMaintenanceInterval = internal_trace_config.sample_rate == 0 ? 60 : internal_trace_config.sample_rate;
         }
       });
     }
@@ -2010,6 +2018,21 @@ void registerDnstapLogger([[maybe_unused]] const DnstapLoggerConfiguration& conf
 #endif
 }
 
+void registerOtlpLogger([[maybe_unused]] const OtlpLoggerConfiguration& config)
+{
+#if !defined(DISABLE_PROTOBUF) && defined(HAVE_LIBCURL)
+  if (dnsdist::configuration::yaml::s_inClientMode || dnsdist::configuration::yaml::s_inConfigCheckMode) {
+    auto object = std::shared_ptr<RemoteLoggerInterface>(nullptr);
+    dnsdist::configuration::yaml::registerType<RemoteLoggerInterface>(object, config.name);
+    return;
+  }
+  std::shared_ptr<RemoteLoggerInterface> object = std::make_shared<OTLPLogger>(std::string(config.address), config.interval, config.queue_size, config.batch_size);
+  dnsdist::configuration::yaml::registerType<RemoteLoggerInterface>(object, config.name);
+#else
+  throw std::runtime_error("Unable to create OTLP logger: OTLP support is disabled");
+#endif /* !defined(DISABLE_PROTOBUF) && defined(HAVE_LIBCURL) */
+}
+
 void registerKVSObjects([[maybe_unused]] const KeyValueStoresConfiguration& config)
 {
 #if defined(HAVE_LMDB) || defined(HAVE_CDB) || defined(HAVE_MMDB)

pdns/dnsdistdist/dnsdist-lua-actions.cc

@@ -29,6 +29,7 @@
 #include "dnsdist-rule-chains.hh"
 #include "dnstap.hh"
 #include "dolog.hh"
+#include "otlp_logger.hh"
 #include "remote_logger.hh"
 #include <memory>
 #include <optional>
@@ -253,9 +254,9 @@ void setupLuaActions(LuaContext& luaCtx)
         if (remote_logger.second != nullptr) {
           // avoids potentially-evaluated-expression warning with clang.
           RemoteLoggerInterface& remoteLoggerRef = *remote_logger.second;
-          if (typeid(remoteLoggerRef) != typeid(RemoteLogger)) {
+          if (typeid(remoteLoggerRef) != typeid(RemoteLogger) && typeid(remoteLoggerRef) != typeid(OTLPLogger)) {
             // We could let the user do what he wants, but wrapping PowerDNS Protobuf inside a FrameStream tagged as dnstap is logically wrong.
-            throw std::runtime_error(std::string("SetTraceAction only takes RemoteLogger."));
+            throw std::runtime_error(std::string("SetTraceAction only takes RemoteLogger or OTLPLogger."));
           }
           loggers.push_back(remote_logger.second);
         }

pdns/dnsdistdist/dnsdist-lua-bindings-protobuf.cc

@@ -28,6 +28,7 @@
 #include "dnsdist-protobuf.hh"
 #include "dnstap.hh"
 #include "fstrm_logger.hh"
+#include "otlp_logger.hh"
 #include "ipcipher.hh"
 #include "remote_logger.hh"
 #include "remote_logger_pool.hh"
@@ -196,6 +197,26 @@ void setupLuaBindingsProtoBuf(LuaContext& luaCtx, bool client, bool configCheck)
 #endif /* HAVE_FSTRM */
   });
 
+  luaCtx.writeFunction("newOtlpLogger", [client, configCheck]([[maybe_unused]] const std::string& address, [[maybe_unused]] std::optional<LuaAssociativeTable<unsigned int>> params) {
+#if !defined(DISABLE_PROTOBUF) && defined(HAVE_LIBCURL)
+    if (client || configCheck) {
+      return std::shared_ptr<RemoteLoggerInterface>(nullptr);
+    }
+    size_t interval{5};
+    size_t batchSize{100};
+    size_t cacheSize{500};
+
+    getOptionalValue<size_t>(params, "interval", interval);
+    getOptionalValue<size_t>(params, "batchSize", batchSize);
+    getOptionalValue<size_t>(params, "queueSize", cacheSize);
+    checkAllParametersConsumed("newOtlpLogger", params);
+
+    return std::shared_ptr<RemoteLoggerInterface>(new OTLPLogger(address, interval, cacheSize, batchSize));
+#else
+    throw std::runtime_error("Protobuf and CURL are required for OTLP remote loggers");
+#endif /* !defined(DISABLE_PROTOBUF) && defined(HAVE_LIBCURL) */
+  });
+
   luaCtx.registerFunction<std::string (std::shared_ptr<RemoteLoggerInterface>::*)() const>("toString", [](const std::shared_ptr<RemoteLoggerInterface>& logger) {
     if (logger) {
       return logger->toString();

pdns/dnsdistdist/dnsdist-lua-configuration-items.cc

@@ -29,6 +29,7 @@
 #include "dnsdist-configuration.hh"
 #include "dnsdist-lua.hh"
 #include "dolog.hh"
+#include "otlp_logger.hh"
 #include "ext/luawrapper/include/LuaContext.hpp"
 
 namespace dnsdist::lua
@@ -197,9 +198,9 @@ static void setupOpenTelemetryConfigurationItems(LuaContext& luaCtx)
         if (remote_logger.second != nullptr) {
           // avoids potentially-evaluated-expression warning with clang.
           RemoteLoggerInterface& remoteLoggerRef = *remote_logger.second;
-          if (typeid(remoteLoggerRef) != typeid(RemoteLogger)) {
+          if (typeid(remoteLoggerRef) != typeid(RemoteLogger) && typeid(remoteLoggerRef) != typeid(OTLPLogger)) {
             // We could let the user do what he wants, but wrapping PowerDNS Protobuf inside a FrameStream tagged as dnstap is logically wrong.
-            throw std::runtime_error(std::string("setOpenTelemetryInternalTrace only takes RemoteLogger."));
+            throw std::runtime_error(std::string("setOpenTelemetryInternalTrace only takes RemoteLogger and OTLPLogger."));
           }
           loggers.push_back(remote_logger.second);
         }

pdns/dnsdistdist/dnsdist-opentelemetry.cc

@@ -24,12 +24,14 @@
 #include "dnsdist-ecs.hh"
 #include "sanitizer.hh"
 
+#include <any>
 #include <memory>
 #include <vector>
 
 #ifndef DISABLE_PROTOBUF
 #include "protozero.hh"
 #include "protozero-trace.hh"
+#include "otlp_logger.hh"
 #endif
 
 namespace pdns::trace::dnsdist
@@ -436,12 +438,31 @@ void sendTracesToRemoteLoggers(const std::shared_ptr<Tracer>& tracer, [[maybe_un
 #ifndef DISABLE_PROTOBUF
   static thread_local string pbBuf;
   pbBuf.clear();
+
+  bool haveNonOTLPLogger = false;
+  for (const auto& remotelogger : remoteloggers) {
+    RemoteLoggerInterface& remoteLoggerRef = *remotelogger;
+    haveNonOTLPLogger = haveNonOTLPLogger || typeid(remoteLoggerRef) != typeid(OTLPLogger);
+    if (haveNonOTLPLogger) {
+      break;
+    }
+  }
+
   pdns::ProtoZero::Message minimalMsg{pbBuf};
-  minimalMsg.setType(pdns::ProtoZero::Message::MessageType::InternalType);
-  minimalMsg.setOpenTelemetryTraceID(tracer->getTraceID());
-  minimalMsg.setOpenTelemetryData(tracer->getOTProtobuf());
+  if (haveNonOTLPLogger) {
+    minimalMsg.setType(pdns::ProtoZero::Message::MessageType::InternalType);
+    minimalMsg.setOpenTelemetryTraceID(tracer->getTraceID());
+    minimalMsg.setOpenTelemetryData(tracer->getOTProtobuf());
+  }
+
   for (const auto& remotelogger : remoteloggers) {
-    remotelogger->queueData(pbBuf);
+    RemoteLoggerInterface& remoteLoggerRef = *remotelogger;
+    if (typeid(remoteLoggerRef) != typeid(OTLPLogger)) {
+      remotelogger->queueData(pbBuf);
+    }
+    else {
+      std::dynamic_pointer_cast<OTLPLogger>(remotelogger)->queueData(tracer->getTracesData());
+    }
   }
 #endif // DISABLE_PROTOBUF
 }

pdns/dnsdistdist/dnsdist-rust-bridge.hh

@@ -32,13 +32,15 @@ struct DNSResponseActionWrapper
 
 struct ProtobufLoggerConfiguration;
 struct DnstapLoggerConfiguration;
+struct OtlpLoggerConfiguration;
 struct KeyValueStoresConfiguration;
 struct MmdbConfiguration;
 struct NetmaskGroupConfiguration;
 struct TimedIpSetConfiguration;
 
 void registerProtobufLogger(const ProtobufLoggerConfiguration& config);
 void registerDnstapLogger(const DnstapLoggerConfiguration& config);
+void registerOtlpLogger(const OtlpLoggerConfiguration& config);
 void registerKVSObjects(const KeyValueStoresConfiguration& config);
 void registerMMDBObjects(const ::rust::Vec<MmdbConfiguration>& config);
 void registerNMGObjects(const ::rust::Vec<NetmaskGroupConfiguration>& nmgs);

pdns/dnsdistdist/dnsdist-rust-lib/rust-middle-in.rs

@@ -14,6 +14,7 @@
         type DNSResponseActionWrapper;
         fn registerProtobufLogger(config: &ProtobufLoggerConfiguration);
         fn registerDnstapLogger(config: &DnstapLoggerConfiguration);
+        fn registerOtlpLogger(config: &OtlpLoggerConfiguration);
         fn registerKVSObjects(config: &KeyValueStoresConfiguration);
         fn registerMMDBObjects(config: &Vec<MmdbConfiguration>);
         fn registerNMGObjects(nmgs: &Vec<NetmaskGroupConfiguration>);

pdns/dnsdistdist/dnsdist-rust-lib/rust-post-in.rs

@@ -54,6 +54,9 @@ fn register_remote_loggers(
   for logger in &config.dnstap_loggers {
     dnsdistsettings::registerDnstapLogger(logger);
   }
+  for logger in &config.otlp_loggers {
+    dnsdistsettings::registerOtlpLogger(logger);
+  }
 }
 
 fn get_global_configuration_from_serde(

pdns/dnsdistdist/dnsdist-settings-definitions.yml

@@ -203,6 +203,10 @@ remote_logging:
       type: "Vec<DnstapLoggerConfiguration>"
       default: true
       description: "List of endpoints to send queries and/or responses data to, using the dnstap format"
+    - name: "otlp_loggers"
+      type: "Vec<OtlpLoggerConfiguration>"
+      default: true
+      description: "List of endpoints to send OpenTelemetry Traces to, using OTLP"
 
 protobuf_logger:
   description: "Endpoint to send queries and/or responses data to, using the native PowerDNS format"
@@ -274,6 +278,28 @@ dnstap_logger:
       default: 1
       description: "Number of connections to open to the endpoint"
 
+otlp_logger:
+  description: "List of endpoints to send OpenTelemetry Traces to, using OTLP."
+  parameters:
+    - name: "name"
+      type: "String"
+      description: "Name of this endpoint"
+    - name: "address"
+      type: "String"
+      description: "The URL of the endpoint. Must contain the scheme (http, https) and the path (like /v1/traces)"
+    - name: "interval"
+      type: "u32"
+      description: "Interval in seconds to wait before sending the next batch of OTLP messages"
+      default: 5
+    - name: "queue_size"
+      type: "u32"
+      description: "Maximum number of traces in the backlog"
+      default: 500
+    - name: "batch_size"
+      type: "u32"
+      description: "Maximum number of traces to send to the endpoint"
+      default: 100
+
 proto_buf_meta:
   description: "Meta-data entry to be added to a Protocol Buffer message"
   parameters:

pdns/dnsdistdist/meson.build

@@ -50,6 +50,7 @@ subdir('meson' / 'net-libs')                # Network Libraries
 subdir('meson' / 'tm-gmtoff')               # Check for tm_gmtoff field in struct tm
 subdir('meson' / 'mmap')                    # Check for mmap
 subdir('meson' / 'libcap')                  # Libcap to drop capabilities
+subdir('meson' / 'libcurl')                 # Libcurl for OTLPLogger
 subdir('meson' / 'libedit')                 # Libedit
 subdir('meson' / 'libsodium')               # Libsodium
 subdir('meson' / 'libcrypto')               # OpenSSL libcrypto
@@ -209,6 +210,7 @@ common_sources += files(
   src_dir / 'libssl.cc',
   src_dir / 'logging.cc',
   src_dir / 'misc.cc',
+  src_dir / 'otlp_logger.cc',
   src_dir / 'protozero.cc',
   src_dir / 'protozero-trace.cc',
   src_dir / 'proxy-protocol.cc',
@@ -231,6 +233,12 @@ conditional_sources = {
     ],
     'condition': dep_cdb.found(),
   },
+  'minicurl': {
+    'sources': [
+      src_dir / 'minicurl.cc',
+    ],
+    'condition': get_option('otlp').allowed() and dep_libcurl.found(),
+  },
   'doq': {
     'sources': [
       src_dir / 'doq.cc',
@@ -395,6 +403,7 @@ deps = [
   dep_ipcrypt2,
   dep_libcap,
   dep_libcrypto,
+  dep_libcurl,
   dep_gnutls,
   dep_libedit,
   dep_json11,

pdns/dnsdistdist/meson/libcurl/meson.build

@@ -0,0 +1,12 @@
+opt_otlp = get_option('otlp')
+opt_libcurl = get_option('libcurl')
+
+if opt_otlp.allowed() and opt_libcurl.allowed()
+  dep_libcurl = dependency('libcurl', version: '>= 7.21.3', required: false)
+else
+  dep_libcurl = dependency('', required: false)
+endif
+
+conf.set('HAVE_LIBCURL', dep_libcurl.found(), description: 'Whether we have libcurl')
+summary('cURL', dep_libcurl.found(), bool_yn: true, section: 'Configuration')
+summary('OTLP remote logger', dep_libcurl.found() and opt_otlp.allowed(), bool_yn: true, section: 'Configuration')

pdns/dnsdistdist/meson_options.txt

@@ -2,6 +2,7 @@ option('dnscrypt', type: 'feature', value: 'disabled', description: 'Enable DNSC
 option('libcap', type: 'feature', value: 'auto', description: 'Enable libcap for capabilities handling')
 option('libedit', type: 'feature', value: 'enabled', description: 'Enable libedit')
 option('libsodium', type: 'feature', value: 'auto', description: 'Enable libsodium')
+option('libcurl', type: 'feature', value: 'auto', description: 'Enable libcurl (for the OTLP remote logger)')
 option('libcrypto', type: 'feature', value: 'auto', description: 'Enable OpenSSL libcrypto)')
 option('libcrypto-path', type: 'string', value: '', description: 'Custom path to find OpenSSL libcrypto')
 option('tls-gnutls', type: 'feature', value: 'auto', description: 'GnuTLS-based TLS')
@@ -28,6 +29,7 @@ option('systemd-service-group', type: 'string', value: 'dnsdist', description: '
 option('auto-var-init', type: 'combo', value: 'disabled', choices: ['zero', 'pattern', 'disabled'], description: 'Enable initialization of automatic variables')
 option('snmp', type: 'feature', value: 'disabled', description: 'Enable SNMP')
 option('dnstap', type: 'feature', value: 'auto', description: 'Enable DNSTAP support through libfstrm')
+option('otlp', type: 'feature', value: 'auto', description: 'Enable OTLP support')
 option('nghttp2', type: 'feature', value: 'auto', description: 'Enable nghttp2 library support for DNS over HTTP/2')
 option('cdb', type: 'feature', value: 'auto', description: 'CDB key-value store support')
 option('lmdb', type: 'feature', value: 'auto', description: 'LMDB key-value store support')

pdns/dnsdistdist/minicurl.cc

@@ -0,0 +1 @@
+../minicurl.cc

pdns/dnsdistdist/minicurl.hh

@@ -0,0 +1 @@
+../minicurl.hh