micromegas/
lib.rs

1//! Micromegas is a unified and scalable observability stack.
2//! It can help you collect and query logs, metrics and traces.
3//!
4//! # Very high level architecture
5//!
6//!
7//! ```text
8//! ┌─────────────────┐
9//! │ rust application│──────▶
10//! └─────────────────┘       ┌─────────┐     ┌───────┐     ┌─────────┐     ┌──────────┐
11//!                           │ingestion│────▶│pg & S3│◀────│analytics│◀────│python API│
12//! ┌─────────────────┐       └─────────┘     └───────┘     └─────────┘     └──────────┘
13//! │ unreal engine   │──────▶
14//! └─────────────────┘
15//!
16//! ```
17//! ## Rust Instrumentation
18//! For rust applications, use micromegas::tracing for minimal overhead. Interoperability with tokio tracing's logs is also enabled by default.
19//!
20//! ## Unreal instrumentation
21//! `MicromegasTracing` should be added to Unreal's Core module and `MicromegasTelemetrySink` can be added to a game or to a high level plugin. See <https://github.com/madesroches/micromegas/tree/main/unreal> for implementation. It has been tested in editor, client and server builds on multiple platforms.
22//!
23//! ## Telemetry ingestion server
24//! <https://github.com/madesroches/micromegas/blob/main/rust/telemetry-ingestion-srv/src/main.rs>
25//!
26//! ## FlightSQL server
27//! <https://github.com/madesroches/micromegas/blob/main/rust/flight-sql-srv/src/flight_sql_srv.rs>
28//!
29//! ## Lakehouse daemon
30//! <https://github.com/madesroches/micromegas/blob/main/rust/telemetry-admin-cli/src/telemetry_admin.rs> (with `crond` argument)
31//!
32//! ## Python API
33//! <https://pypi.org/project/micromegas/>
34//!
35//!
36//! # Local developer configuration
37//!
38//! For testing purposes, you can run the entire stack on your local workstation.
39//!
40//! ## Environment variables
41//!
42//!  - `MICROMEGAS_DB_USERNAME` and `MICROMEGAS_DB_PASSWD`: used by the database configuration script
43//!  - `export MICROMEGAS_TELEMETRY_URL=http://localhost:9000`
44//!  - `export MICROMEGAS_SQL_CONNECTION_STRING=postgres://{uname}:{passwd}@localhost:5432`
45//!  - `export MICROMEGAS_OBJECT_STORE_URI=file:///some/local/path`
46//!
47//! 1. Clone the github repository
48//! ```text
49//! > git clone https://github.com/madesroches/micromegas.git
50//! ```
51//!
52//! 2. Start a local instance of postgresql (requires docker and python)
53//!
54//! ```text
55//! > cd micromegas/local_test_env/db
56//! > ./run.py
57//! ```
58//!
59//! 3. In a new shell, start the ingestion server
60//! ```text
61//! > cd micromegas/rust
62//! > cargo run -p telemetry-ingestion-srv -- --listen-endpoint-http 127.0.0.1:9000
63//! ```
64//!
65//!
66//! 4. In a new shell, start the flightsql server
67//! ```text
68//! > cd micromegas/rust
69//! > cargo run -p flight-sql-srv -- --disable-auth
70//! ```
71//!
72//! 5. In a new shell, start the daemon
73//! ```text
74//! > cd micromegas/rust
75//! > cargo run -p telemetry-admin -- crond
76//! ```
77//!
78//! 6. In a python interpreter, query the analytics service
79//! ```python
80//! # local connection test
81//! import datetime
82//! import micromegas
83//! client = micromegas.connect() #connects to localhost by default
84//! now = datetime.datetime.now(datetime.timezone.utc)
85//! begin = now - datetime.timedelta(days=1)
86//! end = now
87//! sql = """
88//! SELECT *
89//! FROM log_entries
90//! ORDER BY time DESC
91//! LIMIT 10
92//! ;"""
93//! df = client.query(sql, begin, end)
94//! df #dataframe containing the result of the query
95//! ```
96//!
97#![allow(missing_docs)]
98#![allow(clippy::new_without_default)]
99
100/// re-exports (always available)
101pub use chrono;
102pub use uuid;
103
104/// re-exports (server-only)
105#[cfg(feature = "server")]
106pub use arrow_flight;
107#[cfg(feature = "server")]
108pub use axum;
109#[cfg(feature = "server")]
110pub use datafusion;
111#[cfg(feature = "server")]
112pub use object_store;
113#[cfg(feature = "server")]
114pub use prost;
115#[cfg(feature = "server")]
116pub use sqlx;
117#[cfg(feature = "server")]
118pub use tonic;
119
120/// telemetry protocol
121pub mod telemetry {
122    pub use micromegas_telemetry::*;
123}
124
125/// publication of the recorded events using http
126pub mod telemetry_sink {
127    pub use micromegas_telemetry_sink::*;
128}
129
130/// low level tracing - has minimal dependencies
131pub mod tracing {
132    pub use micromegas_tracing::*;
133}
134
135// Re-export proc macros at the top level for easy access
136pub use micromegas_proc_macros::*;
137
138/// authentication providers
139#[cfg(feature = "server")]
140pub mod auth {
141    pub use micromegas_auth::*;
142}
143
144/// records telemetry in data lake
145#[cfg(feature = "server")]
146pub mod ingestion {
147    pub use micromegas_ingestion::*;
148}
149
150/// makes the telemetry data lake accessible and useful
151#[cfg(feature = "server")]
152pub mod analytics {
153    pub use micromegas_analytics::*;
154}
155
156/// perfetto protobufs
157#[cfg(feature = "server")]
158pub mod perfetto {
159    pub use micromegas_perfetto::*;
160}
161
162#[cfg(feature = "server")]
163pub mod servers;
164
165#[cfg(feature = "server")]
166pub mod client;
167
168#[cfg(feature = "server")]
169pub mod utils;
micromegas/lib.rs

micromegas/
lib.rs
