Skip to main content

sketching/
pipeline.rs

1use opentelemetry::{global, trace::TracerProvider as _, KeyValue};
2use opentelemetry_otlp::{tonic_types::metadata::MetadataMap, WithExportConfig, WithTonicConfig};
3use opentelemetry_sdk::{
4    trace::{Sampler, SdkTracerProvider},
5    Resource,
6};
7use opentelemetry_semantic_conventions::{
8    attribute::{DEPLOYMENT_ENVIRONMENT_NAME, SERVICE_VERSION},
9    SCHEMA_URL,
10};
11use std::{str::FromStr, time::Duration};
12use tracing::Subscriber;
13use tracing_core::Level;
14use tracing_opentelemetry::OpenTelemetryLayer;
15use tracing_subscriber::{filter::Directive, prelude::*, EnvFilter, Registry};
16
17const MAX_EVENTS_PER_SPAN: u32 = 64 * 1024;
18const MAX_ATTRIBUTES_PER_SPAN: u32 = 128;
19
20/// This does all the startup things for the logging pipeline
21pub fn start_logging_pipeline(
22    otlp_endpoint: &Option<String>,
23    log_filter: crate::LogLevel,
24) -> Result<(Option<SdkTracerProvider>, Box<dyn Subscriber + Send + Sync>), String> {
25    // Always force the event span to be generated at the correct level, regardless
26    // of what the user set.
27    let kanidmd_core_directives = [
28        Directive::from_str("kanidmd_core::https::trace=info")
29            .map_err(|err| format!("Invalid directive during log setup: {}", err))?,
30        Directive::from_str("kanidmd_core::https::middleware=info")
31            .map_err(|err| format!("Invalid directive during log setup: {}", err))?,
32    ];
33
34    let mut logging_filter = EnvFilter::builder()
35        .with_default_directive(log_filter.into())
36        .parse("")
37        .map_err(|err| format!("Failed to create OTEL logging filter: {}", err))?;
38    for directive in kanidmd_core_directives.iter() {
39        logging_filter = logging_filter.add_directive(directive.clone());
40    }
41    logging_filter = logging_filter
42        // tell the tonic/grpc/h2 layers to trace at warn, so we can see connectivity issues
43        .add_directive(
44            Directive::from_str("tonic=warn")
45                .map_err(|err| format!("Failed to set tonic logging to warn: {}", err))?,
46        )
47        .add_directive(
48            Directive::from_str("hyper=warn")
49                .map_err(|err| format!("Failed to set hyper logging to warn: {}", err))?,
50        )
51        .add_directive(
52            Directive::from_str("h2=warn")
53                .map_err(|err| format!("Failed to set h2 logging to warn: {}", err))?,
54        )
55        .add_directive(
56            Directive::from_str("h2::proto::streams::prioritize=warn").map_err(|err| {
57                format!(
58                    "Failed to set h2::proto::streams::prioritize logging to warn: {}",
59                    err
60                )
61            })?,
62        );
63
64    eprintln!(
65        "Logging filter initialized: {:?}",
66        logging_filter.to_string()
67    );
68
69    // TODO: work out how to do metrics things
70    if let Some(endpoint) = otlp_endpoint {
71        eprintln!(
72            "Attempting to start OTLP logging pipeline endpoint={}",
73            endpoint
74        );
75
76        // setup metadata so we can auth to third-party services
77        let mut tonic_metadata = MetadataMap::new();
78
79        if let Some(headers) = std::env::var_os("OTEL_EXPORTER_OTLP_HEADERS") {
80            let headers = headers.to_string_lossy();
81            for header in headers.split(',') {
82                if let Some((key, value)) = header.split_once('=') {
83                    if !key.is_empty() && key.eq_ignore_ascii_case("authorization") {
84                        if let Ok(header_value) = tonic::metadata::MetadataValue::from_str(value) {
85                            tonic_metadata.insert("authorization", header_value);
86                        } else {
87                            eprintln!(
88                            "Warning: could not parse OTEL_EXPORTER_OTLP_HEADERS environment variable, skipping this: {}", header
89                        );
90                        };
91                    }
92                }
93            }
94        }
95        let otlp_exporter = opentelemetry_otlp::SpanExporter::builder()
96            .with_tonic()
97            .with_endpoint(endpoint)
98            .with_metadata(tonic_metadata)
99            .with_timeout(Duration::from_secs(5))
100            .build()
101            .map_err(|err| err.to_string())?;
102
103        // this env var gets set at build time, if we can pull it, add it to the metadata
104        let git_rev = match option_env!("KANIDM_PKG_COMMIT_REV") {
105            Some(rev) => format!("-{rev}"),
106            None => "".to_string(),
107        };
108
109        let version = format!("{}{}", env!("CARGO_PKG_VERSION"), git_rev);
110        let hostname = gethostname::gethostname();
111        let hostname = hostname.to_string_lossy();
112        let hostname = hostname.to_lowercase();
113
114        let mut resource = Resource::builder().with_schema_url(
115            [
116                // TODO: it'd be really nice to be able to set the instance ID here, from the server UUID so we know *which* instance on this host is logging
117                KeyValue::new(SERVICE_VERSION, version),
118                KeyValue::new(DEPLOYMENT_ENVIRONMENT_NAME, hostname),
119            ],
120            SCHEMA_URL,
121        );
122
123        // only set the service name if it's not already set in the environment because the SDK defaults to "unknown_service"
124        if std::env::var("OTEL_SERVICE_NAME").is_err() {
125            resource = resource.with_service_name("kanidmd");
126        }
127        let resource = resource.build();
128
129        let provider = opentelemetry_sdk::trace::TracerProviderBuilder::default()
130            .with_batch_exporter(otlp_exporter)
131            // we want *everything!*
132            .with_sampler(Sampler::AlwaysOn)
133            .with_max_events_per_span(MAX_EVENTS_PER_SPAN)
134            .with_max_attributes_per_span(MAX_ATTRIBUTES_PER_SPAN)
135            .with_resource(resource)
136            .build();
137
138        let provider_handle = provider.clone();
139
140        global::set_tracer_provider(provider.clone());
141        provider.tracer("tracing-otel-subscriber");
142
143        let registry = tracing_subscriber::registry()
144            .with(
145                tracing_subscriber::filter::LevelFilter::from_level(Level::INFO)
146                    .with_filter(logging_filter.clone()),
147            )
148            .with(
149                OpenTelemetryLayer::new(provider.tracer("tracing-otel-subscriber"))
150                    .with_filter(logging_filter),
151            );
152
153        Ok((Some(provider_handle), Box::new(registry)))
154    } else {
155        let forest_layer = tracing_forest::ForestLayer::default().with_filter(logging_filter);
156        Ok((None, Box::new(Registry::default().with(forest_layer))))
157    }
158}
159
160/// This helps with cleanly shutting down the tracing/logging providers when done,
161/// so we don't lose traces.
162pub struct TracingPipelineGuard(pub Option<SdkTracerProvider>);
163
164impl Drop for TracingPipelineGuard {
165    fn drop(&mut self) {
166        if let Some(provider) = self.0.take() {
167            if let Err(err) = provider.shutdown() {
168                eprintln!("Error shutting down logging pipeline: {}", err);
169            } else {
170                eprintln!("Logging pipeline completed shutdown");
171            }
172        }
173    }
174}