Added loki logging.

That is probably enough of getting obsessive around ops side of things.

Cargo.lock

@@ -413,6 +413,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "encoding_rs"
+version = "0.8.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "errno"
 version = "0.2.8"
@@ -636,6 +645,7 @@ dependencies = [
  "tokio",
  "tower-http",
  "tracing",
+ "tracing-loki",
  "tracing-opentelemetry",
  "tracing-subscriber",
  "twilight-http",
@@ -644,6 +654,7 @@ dependencies = [
  "twilight-model",
  "twilight-util",
  "ureq",
+ "url",
  "uuid",
 ]
 
@@ -804,6 +815,19 @@ dependencies = [
  "tokio-rustls",
 ]
 
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
 [[package]]
 name = "idna"
 version = "0.3.0"
@@ -849,6 +873,12 @@ dependencies = [
  "windows-sys 0.45.0",
 ]
 
+[[package]]
+name = "ipnet"
+version = "2.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
+
 [[package]]
 name = "itertools"
 version = "0.10.5"
@@ -916,6 +946,16 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "loki-api"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f56d36f573486ba7f462b62cbae597fef7d5d93665e7047956b457531b8a1ced"
+dependencies = [
+ "prost",
+ "prost-types",
+]
+
 [[package]]
 name = "matchers"
 version = "0.1.0"
@@ -1323,6 +1363,38 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "prost"
+version = "0.11.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.11.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b"
+dependencies = [
+ "anyhow",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.11.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88"
+dependencies = [
+ "prost",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.23"
@@ -1406,6 +1478,43 @@ version = "0.6.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
 
+[[package]]
+name = "reqwest"
+version = "0.11.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949"
+dependencies = [
+ "base64 0.21.0",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -1662,6 +1771,12 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
+[[package]]
+name = "snap"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831"
+
 [[package]]
 name = "socket2"
 version = "0.4.7"
@@ -2105,6 +2220,27 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-loki"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630f9edbf303c204f36d08d99510bb7b5b2cb6c010146be3aa6537a183907a95"
+dependencies = [
+ "loki-api",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "snap",
+ "tokio",
+ "tokio-stream",
+ "tracing",
+ "tracing-core",
+ "tracing-log",
+ "tracing-serde",
+ "tracing-subscriber",
+ "url",
+]
+
 [[package]]
 name = "tracing-opentelemetry"
 version = "0.18.0"
@@ -2119,6 +2255,16 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "tracing-serde"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.16"
@@ -2377,6 +2523,18 @@ dependencies = [
  "wasm-bindgen-shared",
 ]
 
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.84"
@@ -2548,6 +2706,15 @@ version = "0.42.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
 
+[[package]]
+name = "winreg"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.5.7"

Cargo.toml

@@ -30,6 +30,7 @@ time = "0.3.20"
 tokio = { version = "1.26.0", features = ["full"] }
 tower-http = { version = "0.4.0", features = ["trace"] }
 tracing = "0.1.37"
+tracing-loki = "0.2.2"
 tracing-opentelemetry = "0.18.0"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 twilight-http = "0.15.1"
@@ -37,5 +38,6 @@ twilight-interactions = "0.15.0"
 twilight-mention = "0.15.1"
 twilight-model = "0.15.1"
 twilight-util = { version = "0.15.1", features = ["builder"] }
+url = "2.3.1"
 ureq = { version = "2.6.2", features = ["json"] }
 uuid = "1.3.0"

src/main.rs

@@ -1,4 +1,4 @@
-use std::{net::SocketAddr, str::FromStr};
+use std::{net::SocketAddr, process, str::FromStr};
 
 use axum::{
     extract::State,
@@ -11,9 +11,11 @@ use commands::{get_fact, set_fact, GetFactCommand, SetFactCommand};
 use ed25519_dalek::{Signature, VerifyingKey};
 use serde::Deserialize;
 use sqlx::{postgres::PgPoolOptions, PgPool};
+use tokio::signal;
 use tower_http::trace::TraceLayer;
-use tracing::{Instrument, debug_span};
+use tracing_subscriber::{
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
+    filter::LevelFilter, layer::SubscriberExt, util::SubscriberInitExt, Layer,
+};
 use twilight_http::Client;
 use twilight_interactions::command::{CommandInputData, CommandModel, CreateCommand};
 
@@ -28,19 +30,29 @@ mod database;
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     dotenvy::dotenv().ok();
-    let tracer = opentelemetry_jaeger::new_agent_pipeline().install_simple()?;
+    let tracer = opentelemetry_jaeger::new_agent_pipeline()
+        .with_service_name("god_replacement_product")
+        .install_simple()?;
     let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
+    let (loki_layer, loki_task) = tracing_loki::builder()
+        .label("host", "mine")?
+        .extra_field("pid", format!("{}", process::id()))?
+        .build_url(url::Url::parse("http://127.0.0.1:3100").unwrap())?;
 
     tracing_subscriber::registry()
+        .with(telemetry.with_filter(LevelFilter::DEBUG))
         .with(
-            tracing_subscriber::EnvFilter::try_from_default_env()
+            tracing_subscriber::fmt::layer()
-                .unwrap_or_else(|_| "god_replacement_product=debug,tower_http=debug".into()),
+                .and_then(loki_layer)
+                .with_filter(
+                    tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| {
+                        "god_replacement_product=debug,tower_http=debug".into()
+                    }),
+                ),
         )
-        .with(tracing_subscriber::fmt::layer())
-        .with(telemetry)
         .init();
 
-    let port = listen_port()?;  
+    let port = listen_port()?;
 
     let pg_pool = PgPoolOptions::new()
         .max_connections(5)
@@ -50,21 +62,53 @@ async fn main() -> anyhow::Result<()> {
     sqlx::migrate!().run(&pg_pool).await?;
 
     let app = Router::new()
-        .route("/", post(post_interaction))
+        .route("/api/discord/interactions/", post(post_interaction))
         .with_state(pg_pool)
         .layer(TraceLayer::new_for_http());
     register_commands().await?;
 
     let addr = SocketAddr::from(([127, 0, 0, 1], port));
     tracing::debug!("listening on {}", addr);
+    tokio::spawn(loki_task);
 
     axum::Server::bind(&addr)
         .serve(app.into_make_service())
+        .with_graceful_shutdown(shutdown_signal())
         .await?;
 
+    tracing::info!("server down");
+
+    opentelemetry::global::shutdown_tracer_provider();
+
     Ok(())
 }
 
+async fn shutdown_signal() {
+    let ctrl_c = async {
+        signal::ctrl_c()
+            .await
+            .expect("failed to install Ctrl+C handler");
+    };
+
+    #[cfg(unix)]
+    let terminate = async {
+        signal::unix::signal(signal::unix::SignalKind::terminate())
+            .expect("failed to install signal handler")
+            .recv()
+            .await;
+    };
+
+    #[cfg(not(unix))]
+    let terminate = std::future::pending::<()>();
+
+    tokio::select! {
+        _ = ctrl_c => {},
+        _ = terminate => {},
+    }
+
+    tracing::info!("signal received, starting graceful shutdown");
+}
+
 type InteractionResult = Result<(StatusCode, Json<InteractionResponse>), (StatusCode, String)>;
 
 fn validate_request(headers: HeaderMap, body: String) -> Result<Interaction, (StatusCode, String)> {
@@ -110,7 +154,7 @@ async fn post_interaction(
                 kind: InteractionResponseType::Pong,
                 data: None,
             };
-            
+
             Ok((StatusCode::OK, Json(pong)))
         }
         InteractionType::ApplicationCommand => {
@@ -119,10 +163,7 @@ async fn post_interaction(
                 return not_found();
             };
             let command_input_data = CommandInputData::from(*data.clone());
-            let slash_command_span = debug_span!("discord_slash_command", grs.command.name=data.name.to_owned());
+            tracing::debug!(command_name = ?data.name, "started processing command");
-            slash_command_span.in_scope(|| {
-                tracing::debug!("started processing command");
-            });
             let result = match &*data.name {
                 SetFactCommand::NAME => {
                     let Ok(command_data) = SetFactCommand::from_interaction(command_input_data) else {
@@ -138,7 +179,6 @@ async fn post_interaction(
                         command_data,
                         &pg_pool,
                     )
-                    .instrument(slash_command_span.clone())
                     .await
                     {
                         Ok(response) => Ok((StatusCode::OK, Json(response))),
@@ -153,7 +193,7 @@ async fn post_interaction(
                         return Err((StatusCode::BAD_REQUEST, format!("{0} requires a user.", GetFactCommand::NAME)));
                     };
 
-                    match get_fact(interaction.channel_id, author_id, command_data, &pg_pool).instrument(slash_command_span.clone()).await
+                    match get_fact(interaction.channel_id, author_id, command_data, &pg_pool).await
                     {
                         Ok(response) => Ok((StatusCode::OK, Json(response))),
                         Err(err) => Err(err),
@@ -162,10 +202,7 @@ async fn post_interaction(
                 _ => not_found(),
             };
 
-            slash_command_span.in_scope(|| {
+            tracing::debug!(command_name = ?data.name, "finished processing command");
-                tracing::debug!("finished processing command");
-            });
-
             result
         }
         _ => not_found(),