micromegas_datafusion_extensions/histogram/
variance.rs

1use super::histogram_udaf::{HistogramArray, make_histogram_arrow_type};
2use datafusion::{
3    arrow::{array::Float64Builder, datatypes::DataType},
4    error::DataFusionError,
5    logical_expr::{ColumnarValue, ScalarUDF, Volatility},
6    prelude::*,
7};
8use std::sync::Arc;
9
10fn compute_variance(n: f64, sum: f64, sum_sq: f64) -> f64 {
11    let mean = sum / n;
12    ((sum_sq / n) - (mean * mean)) * (n / (n - 1.0))
13}
14
15fn variance_from_histogram(values: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError> {
16    if values.len() != 1 {
17        return Err(DataFusionError::Execution(
18            "wrong number of arguments to variance_from_histogram".into(),
19        ));
20    }
21
22    let histo_array: HistogramArray = (&values[0]).try_into()?;
23    let mut result_builder = Float64Builder::with_capacity(histo_array.len());
24    for index_histo in 0..histo_array.len() {
25        result_builder.append_value(compute_variance(
26            histo_array.get_count(index_histo)? as f64,
27            histo_array.get_sum(index_histo)?,
28            histo_array.get_sum_sq(index_histo)?,
29        ));
30    }
31
32    Ok(ColumnarValue::Array(Arc::new(result_builder.finish())))
33}
34/// Creates a user-defined function to compute the variance from a histogram.
35pub fn make_variance_from_histogram_udf() -> ScalarUDF {
36    create_udf(
37        "variance_from_histogram",
38        vec![make_histogram_arrow_type()],
39        DataType::Float64,
40        Volatility::Immutable,
41        Arc::new(&variance_from_histogram),
42    )
43}
micromegas_datafusion_extensions/histogram/variance.rs

micromegas_datafusion_extensions/histogram/
variance.rs
