kanidmd_lib/be/

mod.rs

//! The backend. This contains the "low level" storage and query code, which is
//! implemented as a json-like kv document database. This has no rules about content
//! of the server, which are all enforced at higher levels. The role of the backend
//! is to persist content safely to disk, load that content, and execute queries
//! utilising indexes in the most effective way possible.

use crate::be::dbentry::{DbBackup, DbEntry};
use crate::be::dbrepl::DbReplMeta;
use crate::entry::Entry;
use crate::filter::{Filter, FilterPlan, FilterResolved, FilterValidResolved};
use crate::prelude::*;
use crate::repl::cid::Cid;
use crate::repl::proto::ReplCidRange;
use crate::repl::ruv::{
    ReplicationUpdateVector, ReplicationUpdateVectorReadTransaction,
    ReplicationUpdateVectorTransaction, ReplicationUpdateVectorWriteTransaction,
};
use crate::utils::trigraph_iter;
use crate::value::{IndexType, Value};
use concread::cowcell::*;
use hashbrown::{HashMap as Map, HashSet};
use idlset::v2::IDLBitRange;
use idlset::AndNot;
use kanidm_proto::backup::BackupCompression;
use kanidm_proto::internal::{ConsistencyError, OperationError};
use std::collections::BTreeMap;
use std::fs;
use std::io::prelude::*;
use std::io::Read;
use std::ops::DerefMut;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tracing::{trace, trace_span};
use uuid::Uuid;

use flate2::write::GzEncoder;
use flate2::Compression;

pub(crate) mod dbentry;
pub(crate) mod dbrepl;
pub(crate) mod dbvalue;

mod idl_arc_sqlite;
mod idl_sqlite;
pub(crate) mod idxkey;
pub(crate) mod keystorage;

pub(crate) use self::idxkey::{IdxKey, IdxKeyRef, IdxKeyToRef, IdxSlope};
use crate::be::idl_arc_sqlite::{
    IdlArcSqlite, IdlArcSqliteReadTransaction, IdlArcSqliteTransaction,
    IdlArcSqliteWriteTransaction,
};
use kanidm_proto::internal::FsType;

// Currently disabled due to improvements in idlset for intersection handling.
const FILTER_SEARCH_TEST_THRESHOLD: usize = 0;
const FILTER_EXISTS_TEST_THRESHOLD: usize = 0;
const FILTER_SUBSTR_TEST_THRESHOLD: usize = 4;

#[derive(Debug, Clone)]
/// Limits on the resources a single event can consume. These are defined per-event
/// as they are derived from the userAuthToken based on that individual session
pub struct Limits {
    pub unindexed_allow: bool,
    pub search_max_results: usize,
    pub search_max_filter_test: usize,
    pub filter_max_elements: usize,
}

impl Default for Limits {
    fn default() -> Self {
        Limits {
            unindexed_allow: false,
            search_max_results: DEFAULT_LIMIT_SEARCH_MAX_RESULTS as usize,
            search_max_filter_test: DEFAULT_LIMIT_SEARCH_MAX_FILTER_TEST as usize,
            filter_max_elements: DEFAULT_LIMIT_FILTER_MAX_ELEMENTS as usize,
        }
    }
}

impl Limits {
    pub fn unlimited() -> Self {
        Limits {
            unindexed_allow: true,
            search_max_results: usize::MAX >> 1,
            search_max_filter_test: usize::MAX >> 1,
            filter_max_elements: usize::MAX,
        }
    }

    pub fn api_token() -> Self {
        Limits {
            unindexed_allow: false,
            search_max_results: DEFAULT_LIMIT_API_SEARCH_MAX_RESULTS as usize,
            search_max_filter_test: DEFAULT_LIMIT_API_SEARCH_MAX_FILTER_TEST as usize,
            filter_max_elements: DEFAULT_LIMIT_FILTER_MAX_ELEMENTS as usize,
        }
    }
}

/// The result of a key value request containing the list of entry IDs that
/// match the filter/query condition.
#[derive(Debug, Clone)]
pub enum IdList {
    /// The value is not indexed, and must be assumed that all entries may match.
    AllIds,
    /// The index is "fuzzy" like a bloom filter (perhaps superset is a better description) -
    /// it contains all elements that do match, but may have extra elements that don't.
    /// This requires the caller to perform a filter test to assert that all
    /// returned entries match all assertions within the filter.
    Partial(IDLBitRange),
    /// The set was indexed and is below the filter test threshold. This is because it's
    /// now faster to test with the filter than to continue to access indexes at this point.
    /// Like a partial set, this is a super set of the entries that match the query.
    PartialThreshold(IDLBitRange),
    /// The value is indexed and accurately represents the set of entries that precisely match.
    Indexed(IDLBitRange),
}

#[derive(Debug)]
pub struct IdRawEntry {
    id: u64,
    data: Vec<u8>,
}

#[derive(Debug, Clone)]
pub struct IdxMeta {
    pub idxkeys: Map<IdxKey, IdxSlope>,
}

impl IdxMeta {
    pub fn new(idxkeys: Map<IdxKey, IdxSlope>) -> Self {
        IdxMeta { idxkeys }
    }
}

#[derive(Clone)]
pub struct BackendConfig {
    path: PathBuf,
    pool_size: u32,
    db_name: &'static str,
    fstype: FsType,
    // Cachesizes?
    arcsize: Option<usize>,
}

impl BackendConfig {
    pub fn new(
        path: Option<&Path>,
        pool_size: u32,
        fstype: FsType,
        arcsize: Option<usize>,
    ) -> Self {
        BackendConfig {
            pool_size,
            // This means if path is None, that "" implies an sqlite in memory/ram only database.
            path: path.unwrap_or_else(|| Path::new("")).to_path_buf(),
            db_name: "main",
            fstype,
            arcsize,
        }
    }

    pub(crate) fn new_test(db_name: &'static str) -> Self {
        BackendConfig {
            pool_size: 1,
            path: PathBuf::from(""),
            db_name,
            fstype: FsType::Generic,
            arcsize: Some(2048),
        }
    }
}

#[derive(Clone)]
pub struct Backend {
    /// This is the actual datastorage layer.
    idlayer: Arc<IdlArcSqlite>,
    /// This is a copy-on-write cache of the index metadata that has been
    /// extracted from attributes set, in the correct format for the backend
    /// to consume. We use it to extract indexes from entries during write paths
    /// and to allow the front end to know what indexes exist during a read.
    idxmeta: Arc<CowCell<IdxMeta>>,
    /// The current state of the replication update vector. This is effectively a
    /// time series index of the full list of all changelog entries and what entries
    /// that are part of that change.
    ruv: Arc<ReplicationUpdateVector>,
    cfg: BackendConfig,
}

pub struct BackendReadTransaction<'a> {
    idlayer: IdlArcSqliteReadTransaction<'a>,
    idxmeta: CowCellReadTxn<IdxMeta>,
    ruv: ReplicationUpdateVectorReadTransaction<'a>,
}

unsafe impl Sync for BackendReadTransaction<'_> {}

unsafe impl Send for BackendReadTransaction<'_> {}

pub struct BackendWriteTransaction<'a> {
    idlayer: IdlArcSqliteWriteTransaction<'a>,
    idxmeta_wr: CowCellWriteTxn<'a, IdxMeta>,
    ruv: ReplicationUpdateVectorWriteTransaction<'a>,
}

impl IdRawEntry {
    fn into_dbentry(self) -> Result<(u64, DbEntry), OperationError> {
        serde_json::from_slice(self.data.as_slice())
            .map_err(|e| {
                admin_error!(?e, "Serde JSON Error");
                OperationError::SerdeJsonError
            })
            .map(|dbe| (self.id, dbe))
    }

    fn into_entry(self) -> Result<EntrySealedCommitted, OperationError> {
        let db_e = serde_json::from_slice(self.data.as_slice()).map_err(|e| {
            admin_error!(?e, id = %self.id, "Serde JSON Error");
            let raw_str = String::from_utf8_lossy(self.data.as_slice());
            debug!(raw = %raw_str);
            OperationError::SerdeJsonError
        })?;
        // let id = u64::try_from(self.id).map_err(|_| OperationError::InvalidEntryId)?;
        Entry::from_dbentry(db_e, self.id).ok_or(OperationError::CorruptedEntry(self.id))
    }
}

pub trait BackendTransaction {
    type IdlLayerType: IdlArcSqliteTransaction;
    fn get_idlayer(&mut self) -> &mut Self::IdlLayerType;

    type RuvType: ReplicationUpdateVectorTransaction;
    fn get_ruv(&mut self) -> &mut Self::RuvType;

    fn get_idxmeta_ref(&self) -> &IdxMeta;

    /// Recursively apply a filter, transforming into IdList's on the way. This builds a query
    /// execution log, so that it can be examined how an operation proceeded.
    #[allow(clippy::cognitive_complexity)]
    fn filter2idl(
        &mut self,
        filt: &FilterResolved,
        thres: usize,
    ) -> Result<(IdList, FilterPlan), OperationError> {
        Ok(match filt {
            FilterResolved::Eq(attr, value, idx) => {
                if idx.is_some() {
                    // Get the idx_key
                    let idx_key = value.get_idx_eq_key();
                    // Get the idl for this
                    match self
                        .get_idlayer()
                        .get_idl(attr, IndexType::Equality, &idx_key)?
                    {
                        Some(idl) => (
                            IdList::Indexed(idl),
                            FilterPlan::EqIndexed(attr.clone(), idx_key),
                        ),
                        None => (IdList::AllIds, FilterPlan::EqCorrupt(attr.clone())),
                    }
                } else {
                    // Schema believes this is not indexed
                    (IdList::AllIds, FilterPlan::EqUnindexed(attr.clone()))
                }
            }
            FilterResolved::Stw(attr, subvalue, idx)
            | FilterResolved::Enw(attr, subvalue, idx)
            | FilterResolved::Cnt(attr, subvalue, idx) => {
                // Get the idx_key. Not all types support this, so may return "none".
                trace!(?idx, ?subvalue, ?attr);
                if let (true, Some(idx_key)) = (idx.is_some(), subvalue.get_idx_sub_key()) {
                    self.filter2idl_sub(attr, idx_key)?
                } else {
                    // Schema believes this is not indexed
                    (IdList::AllIds, FilterPlan::SubUnindexed(attr.clone()))
                }
            }
            FilterResolved::Pres(attr, idx) => {
                if idx.is_some() {
                    // Get the idl for this
                    match self.get_idlayer().get_idl(attr, IndexType::Presence, "_")? {
                        Some(idl) => (IdList::Indexed(idl), FilterPlan::PresIndexed(attr.clone())),
                        None => (IdList::AllIds, FilterPlan::PresCorrupt(attr.clone())),
                    }
                } else {
                    // Schema believes this is not indexed
                    (IdList::AllIds, FilterPlan::PresUnindexed(attr.clone()))
                }
            }
            FilterResolved::LessThan(attr, _subvalue, idx) => {
                if idx.is_some() {
                    // TODO: Temporary but we get the PRESENCE index for Ordering operations to
                    // reduce the amount of entries we need to filter in memory. In future we need
                    // a true ordering index, but that's a large block of work on it's own. For now
                    // this already helps a lot for in memory processing.
                    match self.get_idlayer().get_idl(attr, IndexType::Presence, "_")? {
                        Some(idl) => (
                            IdList::Partial(idl),
                            FilterPlan::LessThanIndexed(attr.clone()),
                        ),
                        None => (IdList::AllIds, FilterPlan::LessThanCorrupt(attr.clone())),
                    }
                } else {
                    (IdList::AllIds, FilterPlan::LessThanUnindexed(attr.clone()))
                }
            }
            FilterResolved::Or(l, _) => {
                // Importantly if this has no inner elements, this returns
                // an empty list.
                let mut plan = Vec::with_capacity(0);
                let mut result = IDLBitRange::new();
                let mut partial = false;
                let mut threshold = false;
                // For each filter in l
                for f in l.iter() {
                    // get their idls
                    match self.filter2idl(f, thres)? {
                        (IdList::Indexed(idl), fp) => {
                            plan.push(fp);
                            // now union them (if possible)
                            result = result | idl;
                        }
                        (IdList::Partial(idl), fp) => {
                            plan.push(fp);
                            // now union them (if possible)
                            result = result | idl;
                            partial = true;
                        }
                        (IdList::PartialThreshold(idl), fp) => {
                            plan.push(fp);
                            // now union them (if possible)
                            result = result | idl;
                            partial = true;
                            threshold = true;
                        }
                        (IdList::AllIds, fp) => {
                            plan.push(fp);
                            // If we find anything unindexed, the whole term is unindexed.
                            filter_trace!("Term {:?} is AllIds, shortcut return", f);
                            let setplan = FilterPlan::OrUnindexed(plan);
                            return Ok((IdList::AllIds, setplan));
                        }
                    }
                } // end or.iter()
                  // If we got here, every term must have been indexed or partial indexed.
                if partial {
                    if threshold {
                        let setplan = FilterPlan::OrPartialThreshold(plan);
                        (IdList::PartialThreshold(result), setplan)
                    } else {
                        let setplan = FilterPlan::OrPartial(plan);
                        (IdList::Partial(result), setplan)
                    }
                } else {
                    let setplan = FilterPlan::OrIndexed(plan);
                    (IdList::Indexed(result), setplan)
                }
            }
            FilterResolved::And(l, _) => {
                // This algorithm is a little annoying. I couldn't get it to work with iter and
                // folds due to the logic needed ...

                // First, setup the two filter lists. We always apply AndNot after positive
                // and terms.
                let (f_andnot, f_rem): (Vec<_>, Vec<_>) = l.iter().partition(|f| f.is_andnot());

                // We make this an iter, so everything comes off in order. if we used pop it means we
                // pull from the tail, which is the WORST item to start with!
                let mut f_rem_iter = f_rem.iter();

                // Setup the initial result.
                let (mut cand_idl, fp) = match f_rem_iter.next() {
                    Some(f) => self.filter2idl(f, thres)?,
                    None => {
                        filter_warn!(
                            "And filter was empty, or contains only AndNot, can not evaluate."
                        );
                        return Ok((IdList::Indexed(IDLBitRange::new()), FilterPlan::Invalid));
                    }
                };

                // Setup the counter of terms we have left to evaluate.
                // This is used so that we shortcut return ONLY when we really do have
                // more terms remaining.
                let mut f_rem_count = f_rem.len() + f_andnot.len() - 1;

                // Setup the query plan tracker
                let mut plan = vec![fp];

                match &cand_idl {
                    IdList::Indexed(idl) | IdList::Partial(idl) | IdList::PartialThreshold(idl) => {
                        // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                        // But we only do this when there are actually multiple elements in the and,
                        // because an and with 1 element now is FULLY resolved.
                        if idl.below_threshold(thres) && f_rem_count > 0 {
                            let setplan = FilterPlan::AndPartialThreshold(plan);
                            return Ok((IdList::PartialThreshold(idl.clone()), setplan));
                        } else if idl.is_empty() {
                            // Regardless of the input state, if it's empty, this can never
                            // be satisfied, so return we are indexed and complete.
                            let setplan = FilterPlan::AndEmptyCand(plan);
                            return Ok((IdList::Indexed(IDLBitRange::new()), setplan));
                        }
                    }
                    IdList::AllIds => {}
                }

                // Now, for all remaining,
                for f in f_rem_iter {
                    f_rem_count -= 1;
                    let (inter, fp) = self.filter2idl(f, thres)?;
                    plan.push(fp);
                    cand_idl = match (cand_idl, inter) {
                        (IdList::Indexed(ia), IdList::Indexed(ib)) => {
                            let r = ia & ib;
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else if r.is_empty() {
                                // Regardless of the input state, if it's empty, this can never
                                // be satisfied, so return we are indexed and complete.
                                let setplan = FilterPlan::AndEmptyCand(plan);
                                return Ok((IdList::Indexed(IDLBitRange::new()), setplan));
                            } else {
                                IdList::Indexed(r)
                            }
                        }
                        (IdList::Indexed(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::Indexed(ib))
                        | (IdList::Partial(ia), IdList::Partial(ib)) => {
                            let r = ia & ib;
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::Partial(r)
                            }
                        }
                        (IdList::Indexed(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Indexed(ib))
                        | (IdList::PartialThreshold(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::PartialThreshold(ib)) => {
                            let r = ia & ib;
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::PartialThreshold(r)
                            }
                        }
                        (IdList::Indexed(i), IdList::AllIds)
                        | (IdList::AllIds, IdList::Indexed(i))
                        | (IdList::Partial(i), IdList::AllIds)
                        | (IdList::AllIds, IdList::Partial(i)) => IdList::Partial(i),
                        (IdList::PartialThreshold(i), IdList::AllIds)
                        | (IdList::AllIds, IdList::PartialThreshold(i)) => {
                            IdList::PartialThreshold(i)
                        }
                        (IdList::AllIds, IdList::AllIds) => IdList::AllIds,
                    };
                }

                // debug!("partial cand set ==> {:?}", cand_idl);

                for f in f_andnot.iter() {
                    f_rem_count -= 1;
                    let FilterResolved::AndNot(f_in, _) = f else {
                        filter_error!("Invalid server state, a cand filter leaked to andnot set!");
                        return Err(OperationError::InvalidState);
                    };
                    let (inter, fp) = self.filter2idl(f_in, thres)?;
                    // It's an and not, so we need to wrap the plan accordingly.
                    plan.push(FilterPlan::AndNot(Box::new(fp)));
                    cand_idl = match (cand_idl, inter) {
                        (IdList::Indexed(ia), IdList::Indexed(ib)) => {
                            let r = ia.andnot(ib);
                            /*
                            // Don't trigger threshold on and nots if fully indexed.
                            if r.below_threshold(thres) {
                                // When below thres, we have to return partials to trigger the entry_no_match_filter check.
                                return Ok(IdList::PartialThreshold(r));
                            } else {
                                IdList::Indexed(r)
                            }
                            */
                            IdList::Indexed(r)
                        }
                        (IdList::Indexed(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::Indexed(ib))
                        | (IdList::Partial(ia), IdList::Partial(ib)) => {
                            let r = ia.andnot(ib);
                            // DO trigger threshold on partials, because we have to apply the filter
                            // test anyway, so we may as well shortcut at this point.
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::Partial(r)
                            }
                        }
                        (IdList::Indexed(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Indexed(ib))
                        | (IdList::PartialThreshold(ia), IdList::PartialThreshold(ib))
                        | (IdList::PartialThreshold(ia), IdList::Partial(ib))
                        | (IdList::Partial(ia), IdList::PartialThreshold(ib)) => {
                            let r = ia.andnot(ib);
                            // DO trigger threshold on partials, because we have to apply the filter
                            // test anyway, so we may as well shortcut at this point.
                            if r.below_threshold(thres) && f_rem_count > 0 {
                                let setplan = FilterPlan::AndPartialThreshold(plan);
                                return Ok((IdList::PartialThreshold(r), setplan));
                            } else {
                                IdList::PartialThreshold(r)
                            }
                        }

                        (IdList::Indexed(_), IdList::AllIds)
                        | (IdList::AllIds, IdList::Indexed(_))
                        | (IdList::Partial(_), IdList::AllIds)
                        | (IdList::AllIds, IdList::Partial(_))
                        | (IdList::PartialThreshold(_), IdList::AllIds)
                        | (IdList::AllIds, IdList::PartialThreshold(_)) => {
                            // We could actually generate allids here
                            // and then try to reduce the and-not set, but
                            // for now we just return all ids.
                            IdList::AllIds
                        }
                        (IdList::AllIds, IdList::AllIds) => IdList::AllIds,
                    };
                }

                // What state is the final cand idl in?
                let setplan = match cand_idl {
                    IdList::Indexed(_) => FilterPlan::AndIndexed(plan),
                    IdList::Partial(_) | IdList::PartialThreshold(_) => {
                        FilterPlan::AndPartial(plan)
                    }
                    IdList::AllIds => FilterPlan::AndUnindexed(plan),
                };

                // Finally, return the result.
                // debug!("final cand set ==> {:?}", cand_idl);
                (cand_idl, setplan)
            } // end and
            FilterResolved::Inclusion(l, _) => {
                // For inclusion to be valid, every term must have *at least* one element present.
                // This really relies on indexing, and so it's internal only - generally only
                // for fully indexed existence queries, such as from refint.

                // This has a lot in common with an And and Or but not really quite either.
                let mut plan = Vec::with_capacity(0);
                let mut result = IDLBitRange::new();
                // For each filter in l
                for f in l.iter() {
                    // get their idls
                    match self.filter2idl(f, thres)? {
                        (IdList::Indexed(idl), fp) => {
                            plan.push(fp);
                            if idl.is_empty() {
                                // It's empty, so something is missing. Bail fast.
                                filter_trace!("Inclusion is unable to proceed - an empty (missing) item was found!");
                                let setplan = FilterPlan::InclusionIndexed(plan);
                                return Ok((IdList::Indexed(IDLBitRange::new()), setplan));
                            } else {
                                result = result | idl;
                            }
                        }
                        (_, fp) => {
                            plan.push(fp);
                            let setplan = FilterPlan::InclusionInvalid(plan);
                            error!(
                                ?setplan,
                                "Inclusion is unable to proceed - all terms must be fully indexed!"
                            );
                            return Ok((IdList::Partial(IDLBitRange::new()), setplan));
                        }
                    }
                } // end or.iter()
                  // If we got here, every term must have been indexed
                let setplan = FilterPlan::InclusionIndexed(plan);
                (IdList::Indexed(result), setplan)
            }
            // So why does this return empty? Normally we actually process an AndNot in the context
            // of an "AND" query, but if it's used anywhere else IE the root filter, then there is
            // no other set to exclude - therefore it's empty set. Additionally, even in an OR query
            // the AndNot will be skipped as an empty set for the same reason.
            FilterResolved::AndNot(_f, _) => {
                // get the idl for f
                // now do andnot?
                filter_error!("Requested a top level or isolated AndNot, returning empty");
                (IdList::Indexed(IDLBitRange::new()), FilterPlan::Invalid)
            }
            FilterResolved::Invalid(_) => {
                // Indexed since it is always false and we don't want to influence filter testing
                (IdList::Indexed(IDLBitRange::new()), FilterPlan::Invalid)
            }
        })
    }

    fn filter2idl_sub(
        &mut self,
        attr: &Attribute,
        sub_idx_key: String,
    ) -> Result<(IdList, FilterPlan), OperationError> {
        // Now given that idx_key, we will iterate over the possible graphemes.
        let mut grapheme_iter = trigraph_iter(&sub_idx_key);

        // Substrings are always partial because we have to split the keys up
        // and we don't pay attention to starts/ends with conditions. We need
        // the caller to check those conditions manually at run time. This lets
        // the index focus on trigraph indexes only rather than needing to
        // worry about those other bits. In a way substring indexes are "fuzzy".

        let mut idl = match grapheme_iter.next() {
            Some(idx_key) => {
                match self
                    .get_idlayer()
                    .get_idl(attr, IndexType::SubString, idx_key)?
                {
                    Some(idl) => idl,
                    None => return Ok((IdList::AllIds, FilterPlan::SubCorrupt(attr.clone()))),
                }
            }
            None => {
                // If there are no graphemes this means the attempt is for an empty string, so
                // we return an empty result set.
                return Ok((IdList::Indexed(IDLBitRange::new()), FilterPlan::Invalid));
            }
        };

        if idl.len() > FILTER_SUBSTR_TEST_THRESHOLD {
            for idx_key in grapheme_iter {
                // Get the idl for this
                match self
                    .get_idlayer()
                    .get_idl(attr, IndexType::SubString, idx_key)?
                {
                    Some(r_idl) => {
                        // Do an *and* operation between what we found and our working idl.
                        idl = r_idl & idl;
                    }
                    None => {
                        // if something didn't match, then we simply bail out after zeroing the current IDL.
                        idl = IDLBitRange::new();
                    }
                };

                if idl.len() < FILTER_SUBSTR_TEST_THRESHOLD {
                    break;
                }
            }
        } else {
            drop(grapheme_iter);
        }

        // We exhausted the grapheme iter, exit with what we found.
        Ok((
            IdList::Partial(idl),
            FilterPlan::SubIndexed(attr.clone(), sub_idx_key),
        ))
    }

    #[instrument(level = "debug", name = "be::search", skip_all)]
    fn search(
        &mut self,
        erl: &Limits,
        filt: &Filter<FilterValidResolved>,
    ) -> Result<Vec<Arc<EntrySealedCommitted>>, OperationError> {
        // Unlike DS, even if we don't get the index back, we can just pass
        // to the in-memory filter test and be done.

        trace!(filter_optimised = ?filt);

        let (idl, fplan) = trace_span!("be::search -> filter2idl")
            .in_scope(|| self.filter2idl(filt.to_inner(), FILTER_SEARCH_TEST_THRESHOLD))?;

        debug!(search_filter_executed_plan = %fplan);

        match &idl {
            IdList::AllIds => {
                if !erl.unindexed_allow {
                    admin_error!(
                        "filter (search) is fully unindexed, and not allowed by resource limits"
                    );
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::Partial(idl_br) => {
                // if idl_br.len() > erl.search_max_filter_test {
                if !idl_br.below_threshold(erl.search_max_filter_test) {
                    admin_error!("filter (search) is partial indexed and greater than search_max_filter_test allowed by resource limits");
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::PartialThreshold(_) => {
                // Since we opted for this, this is not the fault
                // of the user and we should not penalise them by limiting on partial.
            }
            IdList::Indexed(idl_br) => {
                // We know this is resolved here, so we can attempt the limit
                // check. This has to fold the whole index, but you know, class=pres is
                // indexed ...
                // if idl_br.len() > erl.search_max_results {
                if !idl_br.below_threshold(erl.search_max_results) {
                    admin_error!("filter (search) is indexed and greater than search_max_results allowed by resource limits");
                    return Err(OperationError::ResourceLimit);
                }
            }
        };

        let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
            admin_error!(?e, "get_identry failed");
            e
        })?;

        let mut entries_filtered = match idl {
            IdList::AllIds => trace_span!("be::search<entry::ftest::allids>").in_scope(|| {
                entries
                    .into_iter()
                    .filter(|e| e.entry_match_no_index(filt))
                    .collect()
            }),
            IdList::Partial(_) => trace_span!("be::search<entry::ftest::partial>").in_scope(|| {
                entries
                    .into_iter()
                    .filter(|e| e.entry_match_no_index(filt))
                    .collect()
            }),
            IdList::PartialThreshold(_) => trace_span!("be::search<entry::ftest::thresh>")
                .in_scope(|| {
                    entries
                        .into_iter()
                        .filter(|e| e.entry_match_no_index(filt))
                        .collect()
                }),
            // Since the index fully resolved, we can shortcut the filter test step here!
            IdList::Indexed(_) => {
                filter_trace!("filter (search) was fully indexed 👏");
                entries
            }
        };

        // If the idl was not indexed, apply the resource limit now. Avoid the needless match since the
        // if statement is quick.
        if entries_filtered.len() > erl.search_max_results {
            admin_error!("filter (search) is resolved and greater than search_max_results allowed by resource limits");
            return Err(OperationError::ResourceLimit);
        }

        // Trim any excess capacity if needed
        entries_filtered.shrink_to_fit();

        Ok(entries_filtered)
    }

    /// Given a filter, assert some condition exists.
    /// Basically, this is a specialised case of search, where we don't need to
    /// load any candidates if they match. This is heavily used in uuid
    /// refint and attr uniqueness.
    #[instrument(level = "debug", name = "be::exists", skip_all)]
    fn exists(
        &mut self,
        erl: &Limits,
        filt: &Filter<FilterValidResolved>,
    ) -> Result<bool, OperationError> {
        trace!(filter_optimised = ?filt);

        // Using the indexes, resolve the IdList here, or AllIds.
        // Also get if the filter was 100% resolved or not.
        let (idl, fplan) = trace_span!("be::exists -> filter2idl")
            .in_scope(|| self.filter2idl(filt.to_inner(), FILTER_EXISTS_TEST_THRESHOLD))?;

        debug!(exist_filter_executed_plan = %fplan);

        // Apply limits to the IdList.
        match &idl {
            IdList::AllIds => {
                if !erl.unindexed_allow {
                    admin_error!(
                        "filter (exists) is fully unindexed, and not allowed by resource limits"
                    );
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::Partial(idl_br) => {
                if !idl_br.below_threshold(erl.search_max_filter_test) {
                    admin_error!("filter (exists) is partial indexed and greater than search_max_filter_test allowed by resource limits");
                    return Err(OperationError::ResourceLimit);
                }
            }
            IdList::PartialThreshold(_) => {
                // Since we opted for this, this is not the fault
                // of the user and we should not penalise them.
            }
            IdList::Indexed(_) => {}
        }

        // Now, check the idl -- if it's fully resolved, we can skip this because the query
        // was fully indexed.
        match &idl {
            IdList::Indexed(idl) => Ok(!idl.is_empty()),
            _ => {
                let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
                    admin_error!(?e, "get_identry failed");
                    e
                })?;

                // if not 100% resolved query, apply the filter test.
                let entries_filtered: Vec<_> =
                    trace_span!("be::exists<entry::ftest>").in_scope(|| {
                        entries
                            .into_iter()
                            .filter(|e| e.entry_match_no_index(filt))
                            .collect()
                    });

                Ok(!entries_filtered.is_empty())
            }
        } // end match idl
    }

    fn retrieve_range(
        &mut self,
        ranges: &BTreeMap<Uuid, ReplCidRange>,
    ) -> Result<Vec<Arc<EntrySealedCommitted>>, OperationError> {
        // First pass the ranges to the ruv to resolve to an absolute set of
        // entry id's.

        let idl = self.get_ruv().range_to_idl(ranges);
        // Because of how this works, I think that it's not possible for the idl
        // to have any missing ids.
        //
        // If it was possible, we could just & with allids to remove the extraneous
        // values.

        if idl.is_empty() {
            // return no entries.
            return Ok(Vec::with_capacity(0));
        }

        // Make it an id list fr the backend.
        let id_list = IdList::Indexed(idl);

        self.get_idlayer().get_identry(&id_list).map_err(|e| {
            admin_error!(?e, "get_identry failed");
            e
        })
    }

    fn verify(&mut self) -> Vec<Result<(), ConsistencyError>> {
        self.get_idlayer().verify()
    }

    fn verify_entry_index(&mut self, e: &EntrySealedCommitted) -> Result<(), ConsistencyError> {
        // First, check our references in name2uuid, uuid2spn and uuid2rdn
        if e.mask_recycled_ts().is_some() {
            let e_uuid = e.get_uuid();
            // We only check these on live entries.
            let (n2u_add, n2u_rem) = Entry::idx_name2uuid_diff(None, Some(e));

            let (Some(n2u_set), None) = (n2u_add, n2u_rem) else {
                admin_error!("Invalid idx_name2uuid_diff state");
                return Err(ConsistencyError::BackendIndexSync);
            };

            // If the set.len > 1, check each item.
            n2u_set
                .iter()
                .try_for_each(|name| match self.get_idlayer().name2uuid(name) {
                    Ok(Some(idx_uuid)) => {
                        if idx_uuid == e_uuid {
                            Ok(())
                        } else {
                            admin_error!("Invalid name2uuid state -> incorrect uuid association");
                            Err(ConsistencyError::BackendIndexSync)
                        }
                    }
                    r => {
                        admin_error!(state = ?r, "Invalid name2uuid state");
                        Err(ConsistencyError::BackendIndexSync)
                    }
                })?;

            let spn = e.get_uuid2spn();
            match self.get_idlayer().uuid2spn(e_uuid) {
                Ok(Some(idx_spn)) => {
                    if spn != idx_spn {
                        admin_error!("Invalid uuid2spn state -> incorrect idx spn value");
                        return Err(ConsistencyError::BackendIndexSync);
                    }
                }
                r => {
                    admin_error!(state = ?r, ?e_uuid, "Invalid uuid2spn state");
                    trace!(entry = ?e);
                    return Err(ConsistencyError::BackendIndexSync);
                }
            };

            let rdn = e.get_uuid2rdn();
            match self.get_idlayer().uuid2rdn(e_uuid) {
                Ok(Some(idx_rdn)) => {
                    if rdn != idx_rdn {
                        admin_error!("Invalid uuid2rdn state -> incorrect idx rdn value");
                        return Err(ConsistencyError::BackendIndexSync);
                    }
                }
                r => {
                    admin_error!(state = ?r, "Invalid uuid2rdn state");
                    return Err(ConsistencyError::BackendIndexSync);
                }
            };
        }

        // Check the other entry:attr indexes are valid
        //
        // This is actually pretty hard to check, because we can check a value *should*
        // exist, but not that a value should NOT be present in the index. Thought needed ...

        // Got here? Ok!
        Ok(())
    }

    fn verify_indexes(&mut self) -> Vec<Result<(), ConsistencyError>> {
        let idl = IdList::AllIds;
        let entries = match self.get_idlayer().get_identry(&idl) {
            Ok(s) => s,
            Err(e) => {
                admin_error!(?e, "get_identry failure");
                return vec![Err(ConsistencyError::Unknown)];
            }
        };

        let r = entries.iter().try_for_each(|e| self.verify_entry_index(e));

        if r.is_err() {
            vec![r]
        } else {
            Vec::with_capacity(0)
        }
    }

    fn verify_ruv(&mut self, results: &mut Vec<Result<(), ConsistencyError>>) {
        // The way we verify this is building a whole second RUV and then comparing it.
        let idl = IdList::AllIds;
        let entries = match self.get_idlayer().get_identry(&idl) {
            Ok(ent) => ent,
            Err(e) => {
                results.push(Err(ConsistencyError::Unknown));
                admin_error!(?e, "get_identry failed");
                return;
            }
        };

        self.get_ruv().verify(&entries, results);
    }

    fn backup(
        &mut self,
        dst_path: &Path,
        compression: BackupCompression,
    ) -> Result<(), OperationError> {
        let repl_meta = self.get_ruv().to_db_backup_ruv();

        // load all entries into RAM, may need to change this later
        // if the size of the database compared to RAM is an issue
        let idl = IdList::AllIds;
        let idlayer = self.get_idlayer();
        let raw_entries: Vec<IdRawEntry> = idlayer.get_identry_raw(&idl)?;

        let entries: Result<Vec<DbEntry>, _> = raw_entries
            .iter()
            .map(|id_ent| {
                serde_json::from_slice(id_ent.data.as_slice())
                    .map_err(|_| OperationError::SerdeJsonError) // log?
            })
            .collect();

        let entries = entries?;

        let db_s_uuid = idlayer
            .get_db_s_uuid()
            .and_then(|u| u.ok_or(OperationError::InvalidDbState))?;
        let db_d_uuid = idlayer
            .get_db_d_uuid()
            .and_then(|u| u.ok_or(OperationError::InvalidDbState))?;
        let db_ts_max = idlayer
            .get_db_ts_max()
            .and_then(|u| u.ok_or(OperationError::InvalidDbState))?;

        let keyhandles = idlayer.get_key_handles()?;

        let bak = DbBackup::V5 {
            // remember env is evaled at compile time.
            version: env!("KANIDM_PKG_SERIES").to_string(),
            db_s_uuid,
            db_d_uuid,
            db_ts_max,
            keyhandles,
            repl_meta,
            entries,
        };

        let serialized_entries_str = serde_json::to_string(&bak).map_err(|e| {
            admin_error!(?e, "serde error");
            OperationError::SerdeJsonError
        })?;

        match compression {
            BackupCompression::NoCompression => fs::write(dst_path, serialized_entries_str)
                .map(|_| ())
                .map_err(|e| {
                    admin_error!(?e, "fs::write error");
                    OperationError::FsError
                }),
            BackupCompression::Gzip => {
                let writer = std::fs::File::create(dst_path).map_err(|e| {
                    admin_error!(?e, "File::create error creating {}", dst_path.display());
                    OperationError::FsError
                })?;

                let mut encoder = GzEncoder::new(writer, Compression::best());
                encoder
                    .write_all(serialized_entries_str.as_bytes())
                    .map_err(|e| {
                        admin_error!(?e, "Gzip compression error writing backup");
                        OperationError::FsError
                    })?;
                Ok(())
            }
        }
    }

    fn name2uuid(&mut self, name: &str) -> Result<Option<Uuid>, OperationError> {
        self.get_idlayer().name2uuid(name)
    }

    fn externalid2uuid(&mut self, name: &str) -> Result<Option<Uuid>, OperationError> {
        self.get_idlayer().externalid2uuid(name)
    }

    fn uuid2spn(&mut self, uuid: Uuid) -> Result<Option<Value>, OperationError> {
        self.get_idlayer().uuid2spn(uuid)
    }

    fn uuid2rdn(&mut self, uuid: Uuid) -> Result<Option<String>, OperationError> {
        self.get_idlayer().uuid2rdn(uuid)
    }
}

impl<'a> BackendTransaction for BackendReadTransaction<'a> {
    type IdlLayerType = IdlArcSqliteReadTransaction<'a>;
    type RuvType = ReplicationUpdateVectorReadTransaction<'a>;

    fn get_idlayer(&mut self) -> &mut IdlArcSqliteReadTransaction<'a> {
        &mut self.idlayer
    }

    fn get_ruv(&mut self) -> &mut ReplicationUpdateVectorReadTransaction<'a> {
        &mut self.ruv
    }

    fn get_idxmeta_ref(&self) -> &IdxMeta {
        &self.idxmeta
    }
}

impl BackendReadTransaction<'_> {
    pub fn list_indexes(&mut self) -> Result<Vec<String>, OperationError> {
        self.get_idlayer().list_idxs()
    }

    pub fn list_id2entry(&mut self) -> Result<Vec<(u64, String)>, OperationError> {
        self.get_idlayer().list_id2entry()
    }

    pub fn list_index_content(
        &mut self,
        index_name: &str,
    ) -> Result<Vec<(String, IDLBitRange)>, OperationError> {
        self.get_idlayer().list_index_content(index_name)
    }

    pub fn get_id2entry(&mut self, id: u64) -> Result<(u64, String), OperationError> {
        self.get_idlayer().get_id2entry(id)
    }

    pub fn list_quarantined(&mut self) -> Result<Vec<(u64, String)>, OperationError> {
        self.get_idlayer().list_quarantined()
    }
}

impl<'a> BackendTransaction for BackendWriteTransaction<'a> {
    type IdlLayerType = IdlArcSqliteWriteTransaction<'a>;
    type RuvType = ReplicationUpdateVectorWriteTransaction<'a>;

    fn get_idlayer(&mut self) -> &mut IdlArcSqliteWriteTransaction<'a> {
        &mut self.idlayer
    }

    fn get_ruv(&mut self) -> &mut ReplicationUpdateVectorWriteTransaction<'a> {
        &mut self.ruv
    }

    fn get_idxmeta_ref(&self) -> &IdxMeta {
        &self.idxmeta_wr
    }
}

impl<'a> BackendWriteTransaction<'a> {
    pub(crate) fn get_ruv_write(&mut self) -> &mut ReplicationUpdateVectorWriteTransaction<'a> {
        &mut self.ruv
    }

    #[instrument(level = "debug", name = "be::create", skip_all)]
    pub fn create(
        &mut self,
        cid: &Cid,
        entries: Vec<EntrySealedNew>,
    ) -> Result<Vec<EntrySealedCommitted>, OperationError> {
        if entries.is_empty() {
            admin_error!("No entries provided to BE to create, invalid server call!");
            return Err(OperationError::EmptyRequest);
        }

        // Check that every entry has a change associated
        // that matches the cid?
        entries.iter().try_for_each(|e| {
            if e.get_changestate().contains_tail_cid(cid) {
                Ok(())
            } else {
                admin_error!(
                    "Entry changelog does not contain a change related to this transaction"
                );
                Err(OperationError::ReplEntryNotChanged)
            }
        })?;

        // Now, assign id's to all the new entries.

        let mut id_max = self.idlayer.get_id2entry_max_id()?;
        let c_entries: Vec<_> = entries
            .into_iter()
            .map(|e| {
                id_max += 1;
                e.into_sealed_committed_id(id_max)
            })
            .collect();

        // All good, lets update the RUV.
        // This auto compresses.
        let ruv_idl = IDLBitRange::from_iter(c_entries.iter().map(|e| e.get_id()));

        // We don't need to skip this like in mod since creates always go to the ruv
        self.get_ruv().insert_change(cid, ruv_idl)?;

        self.idlayer.write_identries(c_entries.iter())?;

        self.idlayer.set_id2entry_max_id(id_max);

        // Now update the indexes as required.
        for e in c_entries.iter() {
            self.entry_index(None, Some(e))?
        }

        Ok(c_entries)
    }

    #[instrument(level = "debug", name = "be::create", skip_all)]
    /// This is similar to create, but used in the replication path as it records all
    /// the CID's in the entry to the RUV, but without applying the current CID as
    /// a new value in the RUV. We *do not* want to apply the current CID in the RUV
    /// related to this entry as that could cause an infinite replication loop!
    pub fn refresh(
        &mut self,
        entries: Vec<EntrySealedNew>,
    ) -> Result<Vec<EntrySealedCommitted>, OperationError> {
        if entries.is_empty() {
            admin_error!("No entries provided to BE to create, invalid server call!");
            return Err(OperationError::EmptyRequest);
        }

        // Assign id's to all the new entries.
        let mut id_max = self.idlayer.get_id2entry_max_id()?;
        let c_entries: Vec<_> = entries
            .into_iter()
            .map(|e| {
                id_max += 1;
                e.into_sealed_committed_id(id_max)
            })
            .collect();

        self.idlayer.write_identries(c_entries.iter())?;

        self.idlayer.set_id2entry_max_id(id_max);

        // Update the RUV with all the changestates of the affected entries.
        for e in c_entries.iter() {
            self.get_ruv().update_entry_changestate(e)?;
        }

        // Now update the indexes as required.
        for e in c_entries.iter() {
            self.entry_index(None, Some(e))?
        }

        Ok(c_entries)
    }

    #[instrument(level = "debug", name = "be::modify", skip_all)]
    pub fn modify(
        &mut self,
        cid: &Cid,
        pre_entries: &[Arc<EntrySealedCommitted>],
        post_entries: &[EntrySealedCommitted],
    ) -> Result<(), OperationError> {
        if post_entries.is_empty() || pre_entries.is_empty() {
            admin_error!("No entries provided to BE to modify, invalid server call!");
            return Err(OperationError::EmptyRequest);
        }

        assert_eq!(post_entries.len(), pre_entries.len());

        let post_entries_iter = post_entries.iter().filter(|e| {
            trace!(?cid);
            trace!(changestate = ?e.get_changestate());
            // If True - This means that at least one attribute that *is* replicated was changed
            // on this entry, so we need to update and add this to the RUV!
            //
            // If False - This means that the entry in question was updated but the changes are all
            // non-replicated so we DO NOT update the RUV here!
            e.get_changestate().contains_tail_cid(cid)
        });

        // All good, lets update the RUV.
        // This auto compresses.
        let ruv_idl = IDLBitRange::from_iter(post_entries_iter.map(|e| e.get_id()));

        if !ruv_idl.is_empty() {
            self.get_ruv().insert_change(cid, ruv_idl)?;
        }

        // Now, given the list of id's, update them
        self.get_idlayer().write_identries(post_entries.iter())?;

        // Finally, we now reindex all the changed entries. We do this by iterating and zipping
        // over the set, because we know the list is in the same order.
        pre_entries
            .iter()
            .zip(post_entries.iter())
            .try_for_each(|(pre, post)| self.entry_index(Some(pre.as_ref()), Some(post)))
    }

    #[instrument(level = "debug", name = "be::incremental_prepare", skip_all)]
    pub fn incremental_prepare(
        &mut self,
        entry_meta: &[EntryIncrementalNew],
    ) -> Result<Vec<Arc<EntrySealedCommitted>>, OperationError> {
        let mut ret_entries = Vec::with_capacity(entry_meta.len());
        let id_max_pre = self.idlayer.get_id2entry_max_id()?;
        let mut id_max = id_max_pre;

        for ctx_ent in entry_meta.iter() {
            let ctx_ent_uuid = ctx_ent.get_uuid();
            let idx_key = ctx_ent_uuid.as_hyphenated().to_string();

            let idl =
                self.get_idlayer()
                    .get_idl(&Attribute::Uuid, IndexType::Equality, &idx_key)?;

            let entry = match idl {
                Some(idl) if idl.is_empty() => {
                    // Create the stub entry, we just need it to have an id number
                    // allocated.
                    id_max += 1;

                    let stub_entry = Arc::new(EntrySealedCommitted::stub_sealed_committed_id(
                        id_max, ctx_ent,
                    ));
                    // Now, the stub entry needs to be indexed. If not, uuid2spn
                    // isn't created, so subsequent index diffs don't work correctly.
                    self.entry_index(None, Some(stub_entry.as_ref()))?;

                    // Okay, entry ready to go.
                    stub_entry
                }
                Some(idl) if idl.len() == 1 => {
                    // Get the entry from this idl.
                    let mut entries = self
                        .get_idlayer()
                        .get_identry(&IdList::Indexed(idl))
                        .map_err(|e| {
                            admin_error!(?e, "get_identry failed");
                            e
                        })?;

                    if let Some(entry) = entries.pop() {
                        // Return it.
                        entry
                    } else {
                        error!("Invalid entry state, index was unable to locate entry");
                        return Err(OperationError::InvalidDbState);
                    }
                    // Done, entry is ready to go
                }
                Some(idl) => {
                    // BUG - duplicate uuid!
                    error!(uuid = ?ctx_ent_uuid, "Invalid IDL state, uuid index must have only a single or no values. Contains {:?}", idl);
                    return Err(OperationError::InvalidDbState);
                }
                None => {
                    // BUG - corrupt index.
                    error!(uuid = ?ctx_ent_uuid, "Invalid IDL state, uuid index must be present");
                    return Err(OperationError::InvalidDbState);
                }
            };

            ret_entries.push(entry);
        }

        if id_max != id_max_pre {
            self.idlayer.set_id2entry_max_id(id_max);
        }

        Ok(ret_entries)
    }

    #[instrument(level = "debug", name = "be::incremental_apply", skip_all)]
    pub fn incremental_apply(
        &mut self,
        update_entries: &[(EntrySealedCommitted, Arc<EntrySealedCommitted>)],
        create_entries: Vec<EntrySealedNew>,
    ) -> Result<(), OperationError> {
        // For the values in create_cands, create these with similar code to the refresh
        // path.
        if !create_entries.is_empty() {
            // Assign id's to all the new entries.
            let mut id_max = self.idlayer.get_id2entry_max_id()?;
            let c_entries: Vec<_> = create_entries
                .into_iter()
                .map(|e| {
                    id_max += 1;
                    e.into_sealed_committed_id(id_max)
                })
                .collect();

            self.idlayer.write_identries(c_entries.iter())?;

            self.idlayer.set_id2entry_max_id(id_max);

            // Update the RUV with all the changestates of the affected entries.
            for e in c_entries.iter() {
                self.get_ruv().update_entry_changestate(e)?;
            }

            // Now update the indexes as required.
            for e in c_entries.iter() {
                self.entry_index(None, Some(e))?
            }
        }

        // Otherwise this is a cid-less copy of modify.
        if !update_entries.is_empty() {
            self.get_idlayer()
                .write_identries(update_entries.iter().map(|(up, _)| up))?;

            for (e, _) in update_entries.iter() {
                self.get_ruv().update_entry_changestate(e)?;
            }

            for (post, pre) in update_entries.iter() {
                self.entry_index(Some(pre.as_ref()), Some(post))?
            }
        }

        Ok(())
    }

    #[instrument(level = "debug", name = "be::reap_tombstones", skip_all)]
    pub fn reap_tombstones(&mut self, cid: &Cid, trim_cid: &Cid) -> Result<usize, OperationError> {
        debug_assert!(cid > trim_cid);
        // Mark a new maximum for the RUV by inserting an empty change. This
        // is important to keep the changestate always advancing.
        self.get_ruv().insert_change(cid, IDLBitRange::default())?;

        // We plan to clear the RUV up to this cid. So we need to build an IDL
        // of all the entries we need to examine.
        let idl = self.get_ruv().trim_up_to(trim_cid).map_err(|e| {
            admin_error!(
                ?e,
                "During tombstone cleanup, failed to trim RUV to {:?}",
                trim_cid
            );
            e
        })?;

        let entries = self
            .get_idlayer()
            .get_identry(&IdList::Indexed(idl))
            .map_err(|e| {
                admin_error!(?e, "get_identry failed");
                e
            })?;

        if entries.is_empty() {
            admin_debug!("No entries affected - reap_tombstones operation success");
            return Ok(0);
        }

        // Now that we have a list of entries we need to partition them into
        // two sets. The entries that are tombstoned and ready to reap_tombstones, and
        // the entries that need to have their change logs trimmed.
        //
        // Remember, these tombstones can be reaped because they were tombstoned at time
        // point 'cid', and since we are now "past" that minimum cid, then other servers
        // will also be trimming these out.
        //
        // Note unlike a changelog impl, we don't need to trim changestates here. We
        // only need the RUV trimmed so that we know if other servers are laggin behind!

        // What entries are tombstones and ready to be deleted?

        let (tombstones, leftover): (Vec<_>, Vec<_>) = entries
            .into_iter()
            .partition(|e| e.get_changestate().can_delete(trim_cid));

        let ruv_idls = self.get_ruv().ruv_idls();

        // Assert that anything leftover still either is *alive* OR is a tombstone
        // and has entries in the RUV!

        if !leftover
            .iter()
            .all(|e| e.get_changestate().is_live() || ruv_idls.contains(e.get_id()))
        {
            admin_error!("Left over entries may be orphaned due to missing RUV entries");
            return Err(OperationError::ReplInvalidRUVState);
        }

        // Now setup to reap_tombstones the tombstones. Remember, in the post cleanup, it's could
        // now have been trimmed to a point we can purge them!

        // Assert the id's exist on the entry.
        let id_list: IDLBitRange = tombstones.iter().map(|e| e.get_id()).collect();

        // Ensure nothing here exists in the RUV index, else it means
        // we didn't trim properly, or some other state violation has occurred.
        if !((&ruv_idls & &id_list).is_empty()) {
            admin_error!("RUV still contains entries that are going to be removed.");
            return Err(OperationError::ReplInvalidRUVState);
        }

        // Now, given the list of id's, reap_tombstones them.
        let sz = id_list.len();
        self.get_idlayer().delete_identry(id_list.into_iter())?;

        // Finally, purge the indexes from the entries we removed. These still have
        // indexes due to class=tombstone.
        tombstones
            .iter()
            .try_for_each(|e| self.entry_index(Some(e), None))?;

        Ok(sz)
    }

    #[instrument(level = "debug", name = "be::update_idxmeta", skip_all)]
    pub fn update_idxmeta(&mut self, idxkeys: Vec<IdxKey>) -> Result<(), OperationError> {
        if self.is_idx_slopeyness_generated()? {
            trace!("Indexing slopes available");
        } else {
            warn!("No indexing slopes available. You should consider reindexing to generate these");
        };

        // TODO: I think anytime we update idx meta is when we should reindex in memory
        // indexes.
        // Probably needs to be similar to create_idxs so we iterate over the set of
        // purely in memory idxs.

        // Setup idxkeys here. By default we set these all to "max slope" aka
        // all indexes are "equal" but also worse case unless analysed. If they
        // have been analysed, we can set the slope factor into here.
        let mut idxkeys = idxkeys
            .into_iter()
            .map(|k| self.get_idx_slope(&k).map(|slope| (k, slope)))
            .collect::<Result<Map<_, _>, _>>()?;

        std::mem::swap(&mut self.idxmeta_wr.deref_mut().idxkeys, &mut idxkeys);
        Ok(())
    }

    // Should take a mut index set, and then we write the whole thing back
    // in a single stripe.
    //
    // So we need a cache, which we load indexes into as we do ops, then we
    // modify them.
    //
    // At the end, we flush those cchange outs in a single run.
    // For create this is probably a
    // TODO: Can this be improved?
    #[allow(clippy::cognitive_complexity)]
    fn entry_index(
        &mut self,
        pre: Option<&EntrySealedCommitted>,
        post: Option<&EntrySealedCommitted>,
    ) -> Result<(), OperationError> {
        let (e_uuid, e_id, uuid_same) = match (pre, post) {
            (None, None) => {
                admin_error!("Invalid call to entry_index - no entries provided");
                return Err(OperationError::InvalidState);
            }
            (Some(pre), None) => {
                trace!("Attempting to remove entry indexes");
                (pre.get_uuid(), pre.get_id(), true)
            }
            (None, Some(post)) => {
                trace!("Attempting to create entry indexes");
                (post.get_uuid(), post.get_id(), true)
            }
            (Some(pre), Some(post)) => {
                trace!("Attempting to modify entry indexes");
                assert_eq!(pre.get_id(), post.get_id());
                (
                    post.get_uuid(),
                    post.get_id(),
                    pre.get_uuid() == post.get_uuid(),
                )
            }
        };

        // Update the names/uuid maps. These have to mask out entries
        // that are recycled or tombstones, so these pretend as "deleted"
        // and can trigger correct actions.

        let mask_pre = pre.and_then(|e| e.mask_recycled_ts());
        let mask_pre = if !uuid_same {
            // Okay, so if the uuids are different this is probably from
            // a replication conflict.  We can't just use the normal none/some
            // check from the Entry::idx functions as they only yield partial
            // changes. Because the uuid is changing, we have to treat pre
            // as a deleting entry, regardless of what state post is in.
            let uuid = mask_pre.map(|e| e.get_uuid()).ok_or_else(|| {
                admin_error!("Invalid entry state - possible memory corruption");
                OperationError::InvalidState
            })?;

            let (n2u_add, n2u_rem) = Entry::idx_name2uuid_diff(mask_pre, None);
            // There will never be content to add.
            assert!(n2u_add.is_none());

            let (eid2u_add, eid2u_rem) = Entry::idx_externalid2uuid_diff(mask_pre, None);
            // There will never be content to add.
            assert!(eid2u_add.is_none());

            let u2s_act = Entry::idx_uuid2spn_diff(mask_pre, None);
            let u2r_act = Entry::idx_uuid2rdn_diff(mask_pre, None);

            trace!(?n2u_rem, ?eid2u_rem, ?u2s_act, ?u2r_act,);

            // Write the changes out to the backend
            if let Some(rem) = n2u_rem {
                self.idlayer.write_name2uuid_rem(rem)?
            }

            if let Some(rem) = eid2u_rem {
                self.idlayer.write_externalid2uuid_rem(rem)?
            }

            match u2s_act {
                None => {}
                Some(Ok(k)) => self.idlayer.write_uuid2spn(uuid, Some(k))?,
                Some(Err(_)) => self.idlayer.write_uuid2spn(uuid, None)?,
            }

            match u2r_act {
                None => {}
                Some(Ok(k)) => self.idlayer.write_uuid2rdn(uuid, Some(k))?,
                Some(Err(_)) => self.idlayer.write_uuid2rdn(uuid, None)?,
            }
            // Return none, mask_pre is now completed.
            None
        } else {
            // Return the state.
            mask_pre
        };

        let mask_post = post.and_then(|e| e.mask_recycled_ts());
        let (n2u_add, n2u_rem) = Entry::idx_name2uuid_diff(mask_pre, mask_post);
        let (eid2u_add, eid2u_rem) = Entry::idx_externalid2uuid_diff(mask_pre, mask_post);

        let u2s_act = Entry::idx_uuid2spn_diff(mask_pre, mask_post);
        let u2r_act = Entry::idx_uuid2rdn_diff(mask_pre, mask_post);

        trace!(
            ?n2u_add,
            ?n2u_rem,
            ?eid2u_add,
            ?eid2u_rem,
            ?u2s_act,
            ?u2r_act
        );

        // Write the changes out to the backend
        if let Some(add) = n2u_add {
            self.idlayer.write_name2uuid_add(e_uuid, add)?
        }
        if let Some(rem) = n2u_rem {
            self.idlayer.write_name2uuid_rem(rem)?
        }

        if let Some(add) = eid2u_add {
            self.idlayer.write_externalid2uuid_add(e_uuid, add)?
        }
        if let Some(rem) = eid2u_rem {
            self.idlayer.write_externalid2uuid_rem(rem)?
        }

        match u2s_act {
            None => {}
            Some(Ok(k)) => self.idlayer.write_uuid2spn(e_uuid, Some(k))?,
            Some(Err(_)) => self.idlayer.write_uuid2spn(e_uuid, None)?,
        }

        match u2r_act {
            None => {}
            Some(Ok(k)) => self.idlayer.write_uuid2rdn(e_uuid, Some(k))?,
            Some(Err(_)) => self.idlayer.write_uuid2rdn(e_uuid, None)?,
        }

        // Extremely Cursed - Okay, we know that self.idxmeta will NOT be changed
        // in this function, but we need to borrow self as mut for the caches in
        // get_idl to work. As a result, this causes a double borrow. To work around
        // this we discard the lifetime on idxmeta, because we know that it will
        // remain constant for the life of the operation.

        let idxmeta = unsafe { &(*(&self.idxmeta_wr.idxkeys as *const _)) };

        let idx_diff = Entry::idx_diff(idxmeta, pre, post);

        idx_diff.into_iter()
            .try_for_each(|act| {
                match act {
                    Ok((attr, itype, idx_key)) => {
                        trace!("Adding {:?} idx -> {:?}: {:?}", itype, attr, idx_key);
                        match self.idlayer.get_idl(attr, itype, &idx_key)? {
                            Some(mut idl) => {
                                idl.insert_id(e_id);
                                if cfg!(debug_assertions)
                                    && *attr == Attribute::Uuid && itype == IndexType::Equality {
                                        // This means a duplicate UUID has appeared in the index.
                                        if idl.len() > 1 {
                                            trace!(duplicate_idl = ?idl, ?idx_key);
                                        }
                                        debug_assert!(idl.len() <= 1);
                                }
                                self.idlayer.write_idl(attr, itype, &idx_key, &idl)
                            }
                            None => {
                                warn!(
                                    "WARNING: index {:?} {:?} was not found. YOU MUST REINDEX YOUR DATABASE",
                                    attr, itype
                                );
                                Ok(())
                            }
                        }
                    }
                    Err((attr, itype, idx_key)) => {
                        trace!("Removing {:?} idx -> {:?}: {:?}", itype, attr, idx_key);
                        match self.idlayer.get_idl(attr, itype, &idx_key)? {
                            Some(mut idl) => {
                                idl.remove_id(e_id);
                                if cfg!(debug_assertions) && *attr == Attribute::Uuid && itype == IndexType::Equality {
                                        // This means a duplicate UUID has appeared in the index.
                                        if idl.len() > 1 {
                                            trace!(duplicate_idl = ?idl, ?idx_key);
                                        }
                                        debug_assert!(idl.len() <= 1);
                                }
                                self.idlayer.write_idl(attr, itype, &idx_key, &idl)
                            }
                            None => {
                                warn!(
                                    "WARNING: index {:?} {:?} was not found. YOU MUST REINDEX YOUR DATABASE",
                                    attr, itype
                                );
                                Ok(())
                            }
                        }
                    }
                }
            })
        // End try_for_each
    }

    #[allow(dead_code)]
    fn missing_idxs(&mut self) -> Result<Vec<(Attribute, IndexType)>, OperationError> {
        let idx_table_list = self.get_idlayer().list_idxs()?;

        // Turn the vec to a real set
        let idx_table_set: HashSet<_> = idx_table_list.into_iter().collect();

        let missing: Vec<_> = self
            .idxmeta_wr
            .idxkeys
            .keys()
            .filter_map(|ikey| {
                // what would the table name be?
                let tname = format!("idx_{}_{}", ikey.itype.as_idx_str(), ikey.attr.as_str());
                trace!("Checking for {}", tname);

                if idx_table_set.contains(&tname) {
                    None
                } else {
                    Some((ikey.attr.clone(), ikey.itype))
                }
            })
            .collect();
        Ok(missing)
    }

    fn create_idxs(&mut self) -> Result<(), OperationError> {
        // Create name2uuid and uuid2name
        trace!("Creating index -> name2uuid");
        self.idlayer.create_name2uuid()?;

        trace!("Creating index -> externalid2uuid");
        self.idlayer.create_externalid2uuid()?;

        trace!("Creating index -> uuid2spn");
        self.idlayer.create_uuid2spn()?;

        trace!("Creating index -> uuid2rdn");
        self.idlayer.create_uuid2rdn()?;

        self.idxmeta_wr
            .idxkeys
            .keys()
            .try_for_each(|ikey| self.idlayer.create_idx(&ikey.attr, ikey.itype))
    }

    pub fn upgrade_reindex(&mut self, v: i64) -> Result<(), OperationError> {
        let dbv = self.get_db_index_version()?;
        admin_debug!(?dbv, ?v, "upgrade_reindex");
        if dbv < v {
            self.reindex(false)?;
            self.set_db_index_version(v)
        } else {
            Ok(())
        }
    }

    #[instrument(level = "info", skip_all)]
    pub fn reindex(&mut self, immediate: bool) -> Result<(), OperationError> {
        let notice_immediate = immediate || (cfg!(not(test)) && cfg!(not(debug_assertions)));

        info!(
            immediate = notice_immediate,
            "System reindex: started - this may take a long time!"
        );

        // Purge the idxs
        // TODO: Purge in memory idxs.
        self.idlayer.danger_purge_idxs()?;

        // Using the index metadata on the txn, create all our idx tables
        // TODO: Needs to create all the in memory indexes.
        self.create_idxs()?;

        // Now, we need to iterate over everything in id2entry and index them
        // Future idea: Do this in batches of X amount to limit memory
        // consumption.
        let idl = IdList::AllIds;
        let entries = self.idlayer.get_identry(&idl).inspect_err(|err| {
            error!(?err, "get_identry failure");
        })?;

        let mut count = 0;

        // This is the longest phase of reindexing, so we have a "progress" display here.
        entries
            .iter()
            .try_for_each(|e| {
                if immediate {
                    count += 1;
                    if count % 2500 == 0 {
                        eprint!("{count}");
                    } else if count % 250 == 0 {
                        eprint!(".");
                    }
                }

                self.entry_index(None, Some(e))
            })
            .inspect_err(|err| {
                error!(?err, "reindex failed");
            })?;

        if immediate {
            eprintln!(" done ✅");
        }

        info!(immediate, "Reindexed {count} entries");

        info!("Optimising Indexes: started");
        self.idlayer.optimise_dirty_idls();
        info!("Optimising Indexes: complete ✅");
        info!("Calculating Index Optimisation Slopes: started");
        self.idlayer.analyse_idx_slopes().inspect_err(|err| {
            error!(?err, "index optimisation failed");
        })?;
        info!("Calculating Index Optimisation Slopes: complete ✅");
        info!("System reindex: complete 🎉");
        Ok(())
    }

    /// ⚠️  - This function will destroy all indexes in the database.
    ///
    /// It should only be called internally by the backend in limited and
    /// specific situations.
    fn danger_purge_idxs(&mut self) -> Result<(), OperationError> {
        self.get_idlayer().danger_purge_idxs()
    }

    /// ⚠️  - This function will destroy all entries and indexes in the database.
    ///
    /// It should only be called internally by the backend in limited and
    /// specific situations.
    pub(crate) fn danger_delete_all_db_content(&mut self) -> Result<(), OperationError> {
        self.get_ruv().clear();
        self.get_idlayer()
            .danger_purge_id2entry()
            .and_then(|_| self.danger_purge_idxs())
    }

    #[cfg(test)]
    pub fn load_test_idl(
        &mut self,
        attr: &Attribute,
        itype: IndexType,
        idx_key: &str,
    ) -> Result<Option<IDLBitRange>, OperationError> {
        self.get_idlayer().get_idl(attr, itype, idx_key)
    }

    fn is_idx_slopeyness_generated(&mut self) -> Result<bool, OperationError> {
        self.get_idlayer().is_idx_slopeyness_generated()
    }

    fn get_idx_slope(&mut self, ikey: &IdxKey) -> Result<IdxSlope, OperationError> {
        // Do we have the slopeyness?
        let slope = self
            .get_idlayer()
            .get_idx_slope(ikey)?
            .unwrap_or_else(|| get_idx_slope_default(ikey));
        trace!("index slope - {:?} -> {:?}", ikey, slope);
        Ok(slope)
    }

    pub fn restore(&mut self, src_path: &Path) -> Result<(), OperationError> {
        let compression = BackupCompression::identify_file(src_path);

        debug!("Backup compression identified: {}", compression);
        let serialized_string = match compression {
            BackupCompression::NoCompression => fs::read_to_string(src_path).map_err(|e| {
                error!("Failed to read backup file {} {e:?}", src_path.display());
                OperationError::FsError
            })?,
            BackupCompression::Gzip => {
                let reader = std::fs::File::open(src_path).map_err(|e| {
                    error!("Failed to open backup file {} {e:?}", src_path.display());
                    OperationError::FsError
                })?;
                let mut decoder = flate2::read::GzDecoder::new(reader);
                let mut decompressed_data = String::new();
                decoder
                    .read_to_string(&mut decompressed_data)
                    .map_err(|e| {
                        error!("GZip decompression error {:?}", e);
                        OperationError::FsError
                    })?;
                decompressed_data
            }
        };

        self.danger_delete_all_db_content().map_err(|e| {
            error!("delete_all_db_content failed {:?}", e);
            e
        })?;

        let idlayer = self.get_idlayer();
        // load all entries into RAM, may need to change this later
        // if the size of the database compared to RAM is an issue

        let dbbak_option: Result<DbBackup, serde_json::Error> =
            serde_json::from_str(&serialized_string);

        let dbbak = dbbak_option.map_err(|e| {
            admin_error!("serde_json error {:?}", e);
            OperationError::SerdeJsonError
        })?;

        let (dbentries, repl_meta, maybe_version) = match dbbak {
            DbBackup::V1(dbentries) => (dbentries, None, None),
            DbBackup::V2 {
                db_s_uuid,
                db_d_uuid,
                db_ts_max,
                entries,
            } => {
                // Do stuff.
                idlayer.write_db_s_uuid(db_s_uuid)?;
                idlayer.write_db_d_uuid(db_d_uuid)?;
                idlayer.set_db_ts_max(db_ts_max)?;
                (entries, None, None)
            }
            DbBackup::V3 {
                db_s_uuid,
                db_d_uuid,
                db_ts_max,
                keyhandles,
                entries,
            } => {
                // Do stuff.
                idlayer.write_db_s_uuid(db_s_uuid)?;
                idlayer.write_db_d_uuid(db_d_uuid)?;
                idlayer.set_db_ts_max(db_ts_max)?;
                idlayer.set_key_handles(keyhandles)?;
                (entries, None, None)
            }
            DbBackup::V4 {
                db_s_uuid,
                db_d_uuid,
                db_ts_max,
                keyhandles,
                repl_meta,
                entries,
            } => {
                // Do stuff.
                idlayer.write_db_s_uuid(db_s_uuid)?;
                idlayer.write_db_d_uuid(db_d_uuid)?;
                idlayer.set_db_ts_max(db_ts_max)?;
                idlayer.set_key_handles(keyhandles)?;
                (entries, Some(repl_meta), None)
            }
            DbBackup::V5 {
                version,
                db_s_uuid,
                db_d_uuid,
                db_ts_max,
                keyhandles,
                repl_meta,
                entries,
            } => {
                // Do stuff.
                idlayer.write_db_s_uuid(db_s_uuid)?;
                idlayer.write_db_d_uuid(db_d_uuid)?;
                idlayer.set_db_ts_max(db_ts_max)?;
                idlayer.set_key_handles(keyhandles)?;
                (entries, Some(repl_meta), Some(version))
            }
        };

        if let Some(version) = maybe_version {
            if version != env!("KANIDM_PKG_SERIES") {
                error!("The provided backup data is from server version {} and is unable to be restored on this instance ({})", version, env!("KANIDM_PKG_SERIES"));
                return Err(OperationError::DB0001MismatchedRestoreVersion);
            }
        } else {
            error!("The provided backup data is from an older server version and is unable to be restored.");
            return Err(OperationError::DB0002MismatchedRestoreVersion);
        };

        // Rebuild the RUV from the backup.
        match repl_meta {
            Some(DbReplMeta::V1 { ruv: db_ruv }) => {
                self.get_ruv()
                    .restore(db_ruv.into_iter().map(|db_cid| db_cid.into()))?;
            }
            None => {
                warn!("Unable to restore replication metadata, this server may need a refresh.");
            }
        }

        info!("Restoring {} entries ...", dbentries.len());

        // Now, we setup all the entries with new ids.
        let mut id_max = 0;
        let identries: Result<Vec<IdRawEntry>, _> = dbentries
            .iter()
            .map(|e| {
                id_max += 1;
                let data = serde_json::to_vec(&e).map_err(|_| OperationError::SerdeCborError)?;
                Ok(IdRawEntry { id: id_max, data })
            })
            .collect();

        let idlayer = self.get_idlayer();

        idlayer.write_identries_raw(identries?.into_iter())?;

        info!("Restored {} entries", dbentries.len());

        let vr = self.verify();
        if vr.is_empty() {
            Ok(())
        } else {
            Err(OperationError::ConsistencyError(
                vr.into_iter().filter_map(|v| v.err()).collect(),
            ))
        }
    }

    /// If any RUV elements are present in the DB, load them now. This provides us with
    /// the RUV boundaries and change points from previous operations of the server, so
    /// that ruv_rebuild can "fill in" the gaps.
    ///
    /// # SAFETY
    ///
    /// Note that you should only call this function during the server startup
    /// to reload the RUV data from the entries of the database.
    ///
    /// Before calling this, the in memory ruv MUST be clear.
    #[instrument(level = "debug", name = "be::ruv_rebuild", skip_all)]
    fn ruv_reload(&mut self) -> Result<(), OperationError> {
        let idlayer = self.get_idlayer();

        let db_ruv = idlayer.get_db_ruv()?;

        // Setup the CID's that existed previously. We don't need to know what entries
        // they affect, we just need them to ensure that we have ranges for replication
        // comparison to take effect properly.
        self.get_ruv().restore(db_ruv)?;

        // Then populate the RUV with the data from the entries.
        self.ruv_rebuild()
    }

    #[instrument(level = "debug", name = "be::ruv_rebuild", skip_all)]
    fn ruv_rebuild(&mut self) -> Result<(), OperationError> {
        // Rebuild the ruv!
        // For now this has to read from all the entries in the DB, but in the future
        // we'll actually store this properly (?). If it turns out this is really fast
        // we may just rebuild this always on startup.

        // NOTE: An important detail is that we don't rely on indexes here!

        let idl = IdList::AllIds;
        let entries = self.get_idlayer().get_identry(&idl).map_err(|e| {
            admin_error!(?e, "get_identry failed");
            e
        })?;

        self.get_ruv().rebuild(&entries)?;

        Ok(())
    }

    pub fn quarantine_entry(&mut self, id: u64) -> Result<(), OperationError> {
        self.get_idlayer().quarantine_entry(id)?;
        // We have to set the index version to 0 so that on next start we force
        // a reindex to automatically occur.
        self.set_db_index_version(0)
    }

    pub fn restore_quarantined(&mut self, id: u64) -> Result<(), OperationError> {
        self.get_idlayer().restore_quarantined(id)?;
        // We have to set the index version to 0 so that on next start we force
        // a reindex to automatically occur.
        self.set_db_index_version(0)
    }

    #[cfg(any(test, debug_assertions))]
    pub fn clear_cache(&mut self) -> Result<(), OperationError> {
        self.get_idlayer().clear_cache()
    }

    pub fn commit(self) -> Result<(), OperationError> {
        let BackendWriteTransaction {
            mut idlayer,
            idxmeta_wr,
            ruv,
        } = self;

        // write the ruv content back to the db.
        idlayer.write_db_ruv(ruv.added(), ruv.removed())?;

        idlayer.commit().map(|()| {
            ruv.commit();
            idxmeta_wr.commit();
        })
    }

    pub(crate) fn reset_db_s_uuid(&mut self) -> Result<Uuid, OperationError> {
        // The value is missing. Generate a new one and store it.
        let nsid = Uuid::new_v4();
        self.get_idlayer().write_db_s_uuid(nsid).map_err(|err| {
            error!(?err, "Unable to persist server uuid");
            err
        })?;
        Ok(nsid)
    }
    pub fn get_db_s_uuid(&mut self) -> Result<Uuid, OperationError> {
        let res = self.get_idlayer().get_db_s_uuid().map_err(|err| {
            error!(?err, "Failed to read server uuid");
            err
        })?;
        match res {
            Some(s_uuid) => Ok(s_uuid),
            None => self.reset_db_s_uuid(),
        }
    }

    /// This generates a new domain UUID and stores it into the database,
    /// returning the new UUID
    fn reset_db_d_uuid(&mut self) -> Result<Uuid, OperationError> {
        let nsid = Uuid::new_v4();
        self.get_idlayer().write_db_d_uuid(nsid).map_err(|err| {
            error!(?err, "Unable to persist domain uuid");
            err
        })?;
        Ok(nsid)
    }

    /// Manually set a new domain UUID and store it into the DB. This is used
    /// as part of a replication refresh.
    pub fn set_db_d_uuid(&mut self, nsid: Uuid) -> Result<(), OperationError> {
        self.get_idlayer().write_db_d_uuid(nsid)
    }

    /// This pulls the domain UUID from the database
    pub fn get_db_d_uuid(&mut self) -> Result<Uuid, OperationError> {
        let res = self.get_idlayer().get_db_d_uuid().map_err(|err| {
            error!(?err, "Failed to read domain uuid");
            err
        })?;
        match res {
            Some(d_uuid) => Ok(d_uuid),
            None => self.reset_db_d_uuid(),
        }
    }

    pub fn set_db_ts_max(&mut self, ts: Duration) -> Result<(), OperationError> {
        self.get_idlayer().set_db_ts_max(ts)
    }

    pub fn get_db_ts_max(&mut self, ts: Duration) -> Result<Duration, OperationError> {
        // if none, return ts. If found, return it.
        match self.get_idlayer().get_db_ts_max()? {
            Some(dts) => Ok(dts),
            None => Ok(ts),
        }
    }

    fn get_db_index_version(&mut self) -> Result<i64, OperationError> {
        self.get_idlayer().get_db_index_version()
    }

    fn set_db_index_version(&mut self, v: i64) -> Result<(), OperationError> {
        self.get_idlayer().set_db_index_version(v)
    }
}

// We have a number of hardcoded, "obvious" slopes that should
// exist. We return these when the analysis has not been run, as
// these are values that are generally "good enough" for most applications
fn get_idx_slope_default(ikey: &IdxKey) -> IdxSlope {
    match (ikey.attr.as_str(), &ikey.itype) {
        (ATTR_NAME, IndexType::Equality)
        | (ATTR_SPN, IndexType::Equality)
        | (ATTR_UUID, IndexType::Equality) => 1,
        (ATTR_CLASS, IndexType::Equality) => 180,
        (_, IndexType::Equality) => 45,
        (_, IndexType::SubString) => 90,
        (_, IndexType::Presence) => 90,
        (_, IndexType::Ordering) => 120,
    }
}

// In the future this will do the routing between the chosen backends etc.
impl Backend {
    #[instrument(level = "debug", name = "be::new", skip_all)]
    pub fn new(
        mut cfg: BackendConfig,
        // path: &str,
        // mut pool_size: u32,
        // fstype: FsType,
        idxkeys: Vec<IdxKey>,
        vacuum: bool,
    ) -> Result<Self, OperationError> {
        debug!(db_tickets = ?cfg.pool_size, profile = %env!("KANIDM_PROFILE_NAME"), cpu_flags = %env!("KANIDM_CPU_FLAGS"));

        // If in memory, reduce pool to 1
        if cfg.path.as_os_str().is_empty() {
            cfg.pool_size = 1;
        }

        // Setup idxkeys here. By default we set these all to "max slope" aka
        // all indexes are "equal" but also worse case unless analysed.
        //
        // During startup this will be "fixed" as the schema core will call reload_idxmeta
        // which will trigger a reload of the analysis data (if present).
        let idxkeys: Map<_, _> = idxkeys
            .into_iter()
            .map(|ikey| {
                let slope = get_idx_slope_default(&ikey);
                (ikey, slope)
            })
            .collect();

        // Load the replication update vector here. Initially we build an in memory
        // RUV, and then we load it from the DB.
        let ruv = Arc::new(ReplicationUpdateVector::default());

        // this has a ::memory() type, but will path == "" work?
        let idlayer = Arc::new(IdlArcSqlite::new(&cfg, vacuum)?);
        let be = Backend {
            cfg,
            idlayer,
            ruv,
            idxmeta: Arc::new(CowCell::new(IdxMeta::new(idxkeys))),
        };

        // Now complete our setup with a txn
        // In this case we can use an empty idx meta because we don't
        // access any parts of
        // the indexing subsystem here.
        let mut idl_write = be.idlayer.write()?;
        idl_write
            .setup()
            .and_then(|_| idl_write.commit())
            .map_err(|e| {
                admin_error!(?e, "Failed to setup idlayer");
                e
            })?;

        // Load/generate any in memory indexes.
        // I think here we don't actually care about in memory indexes until
        // later?

        // Now rebuild the ruv.
        let mut be_write = be.write()?;
        be_write
            .ruv_reload()
            .and_then(|_| be_write.commit())
            .map_err(|e| {
                admin_error!(?e, "Failed to reload ruv");
                e
            })?;

        Ok(be)
    }

    pub fn get_pool_size(&self) -> u32 {
        debug_assert!(self.cfg.pool_size > 0);
        self.cfg.pool_size
    }

    pub fn try_quiesce(&self) {
        self.idlayer.try_quiesce();
    }

    pub fn read(&self) -> Result<BackendReadTransaction<'_>, OperationError> {
        Ok(BackendReadTransaction {
            idlayer: self.idlayer.read()?,
            idxmeta: self.idxmeta.read(),
            ruv: self.ruv.read(),
        })
    }

    pub fn write(&self) -> Result<BackendWriteTransaction<'_>, OperationError> {
        Ok(BackendWriteTransaction {
            idlayer: self.idlayer.write()?,
            idxmeta_wr: self.idxmeta.write(),
            ruv: self.ruv.write(),
        })
    }
}

// What are the possible actions we'll receive here?

#[cfg(test)]
mod tests {
    use super::super::entry::{Entry, EntryInit, EntryNew};
    use super::Limits;
    use super::{
        Backend, BackendConfig, BackendTransaction, BackendWriteTransaction, DbBackup, IdList,
        IdxKey, OperationError,
    };
    use crate::prelude::*;
    use crate::repl::cid::Cid;
    use crate::value::{IndexType, PartialValue, Value};
    use idlset::v2::IDLBitRange;
    use kanidm_proto::backup::BackupCompression;
    use std::fs;
    use std::iter::FromIterator;
    use std::path::Path;
    use std::sync::Arc;
    use std::time::Duration;

    lazy_static! {
        static ref CID_ZERO: Cid = Cid::new_zero();
        static ref CID_ONE: Cid = Cid::new_count(1);
        static ref CID_TWO: Cid = Cid::new_count(2);
        static ref CID_THREE: Cid = Cid::new_count(3);
        static ref CID_ADV: Cid = Cid::new_count(10);
    }

    macro_rules! run_test {
        ($test_fn:expr) => {{
            sketching::test_init();

            // This is a demo idxmeta, purely for testing.
            let idxmeta = vec![
                IdxKey {
                    attr: Attribute::Name.into(),
                    itype: IndexType::Equality,
                },
                IdxKey {
                    attr: Attribute::Name.into(),
                    itype: IndexType::Presence,
                },
                IdxKey {
                    attr: Attribute::Name.into(),
                    itype: IndexType::SubString,
                },
                IdxKey {
                    attr: Attribute::Uuid.into(),
                    itype: IndexType::Equality,
                },
                IdxKey {
                    attr: Attribute::Uuid.into(),
                    itype: IndexType::Presence,
                },
                IdxKey {
                    attr: Attribute::TestAttr.into(),
                    itype: IndexType::Equality,
                },
                IdxKey {
                    attr: Attribute::TestNumber.into(),
                    itype: IndexType::Equality,
                },
            ];

            let be = Backend::new(BackendConfig::new_test("main"), idxmeta, false)
                .expect("Failed to setup backend");

            let mut be_txn = be.write().unwrap();

            let r = $test_fn(&mut be_txn);
            // Commit, to guarantee it worked.
            assert!(be_txn.commit().is_ok());
            r
        }};
    }

    macro_rules! entry_exists {
        ($be:expr, $ent:expr) => {{
            let ei = $ent.clone().into_sealed_committed();
            let filt = ei
                .filter_from_attrs(&[Attribute::Uuid.into()])
                .expect("failed to generate filter")
                .into_valid_resolved();
            let lims = Limits::unlimited();
            let entries = $be.search(&lims, &filt).expect("failed to search");
            entries.first().is_some()
        }};
    }

    macro_rules! entry_attr_pres {
        ($be:expr, $ent:expr, $attr:expr) => {{
            let ei = $ent.clone().into_sealed_committed();
            let filt = ei
                .filter_from_attrs(&[Attribute::UserId.into()])
                .expect("failed to generate filter")
                .into_valid_resolved();
            let lims = Limits::unlimited();
            let entries = $be.search(&lims, &filt).expect("failed to search");
            match entries.first() {
                Some(ent) => ent.attribute_pres($attr),
                None => false,
            }
        }};
    }

    macro_rules! idl_state {
        ($be:expr, $attr:expr, $itype:expr, $idx_key:expr, $expect:expr) => {{
            let t_idl = $be
                .load_test_idl(&$attr, $itype, &$idx_key.to_string())
                .expect("IdList Load failed");
            let t = $expect.map(|v: Vec<u64>| IDLBitRange::from_iter(v));
            assert_eq!(t_idl, t);
        }};
    }

    #[test]
    fn test_be_simple_create() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Create");

            let empty_result = be.create(&CID_ZERO, Vec::with_capacity(0));
            trace!("{:?}", empty_result);
            assert_eq!(empty_result, Err(OperationError::EmptyRequest));

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava(Attribute::UserId, Value::from("william"));
            e.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            let e = e.into_sealed_new();

            let single_result = be.create(&CID_ZERO, vec![e.clone()]);

            assert!(single_result.is_ok());

            // Construct a filter
            assert!(entry_exists!(be, e));
        });
    }

    #[test]
    fn test_be_simple_search() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Search");

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava(Attribute::UserId, Value::from("claire"));
            e.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            let e = e.into_sealed_new();

            let single_result = be.create(&CID_ZERO, vec![e]);
            assert!(single_result.is_ok());
            // Test a simple EQ search

            let filt = filter_resolved!(f_eq(Attribute::UserId, PartialValue::new_utf8s("claire")));

            let lims = Limits::unlimited();

            let r = be.search(&lims, &filt);
            assert!(r.expect("Search failed!").len() == 1);

            // Test empty search

            // Test class pres

            // Search with no results
        });
    }

    #[test]
    fn test_be_search_with_invalid() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Search");

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava(Attribute::UserId, Value::from("bagel"));
            e.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            let e = e.into_sealed_new();

            let single_result = be.create(&CID_ZERO, vec![e]);
            assert!(single_result.is_ok());

            // Test Search with or condition including invalid attribute
            let filt = filter_resolved!(f_or(vec![
                f_eq(Attribute::UserId, PartialValue::new_utf8s("bagel")),
                f_invalid(Attribute::UserId)
            ]));

            let lims = Limits::unlimited();

            let r = be.search(&lims, &filt);
            assert!(r.expect("Search failed!").len() == 1);

            // Test Search with or condition including invalid attribute
            let filt = filter_resolved!(f_and(vec![
                f_eq(Attribute::UserId, PartialValue::new_utf8s("bagel")),
                f_invalid(Attribute::UserId)
            ]));

            let lims = Limits::unlimited();

            let r = be.search(&lims, &filt);
            assert!(r.expect("Search failed!").is_empty());
        });
    }

    #[test]
    fn test_be_simple_modify() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Modify");
            let lims = Limits::unlimited();
            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::UserId, Value::from("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::UserId, Value::from("alice"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"),
            );

            let ve1 = e1.clone().into_sealed_new();
            let ve2 = e2.clone().into_sealed_new();

            assert!(be.create(&CID_ZERO, vec![ve1, ve2]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));

            // You need to now retrieve the entries back out to get the entry id's
            let mut results = be
                .search(&lims, &filter_resolved!(f_pres(Attribute::UserId)))
                .expect("Failed to search");

            // Get these out to usable entries.
            let r1 = results.remove(0);
            let r2 = results.remove(0);

            let mut r1 = r1.as_ref().clone().into_invalid();
            let mut r2 = r2.as_ref().clone().into_invalid();

            // Modify no id (err)
            // This is now impossible due to the state machine design.
            // However, with some unsafe ....
            let ue1 = e1.clone().into_sealed_committed();
            assert!(be
                .modify(&CID_ZERO, &[Arc::new(ue1.clone())], &[ue1])
                .is_err());
            // Modify none
            assert!(be.modify(&CID_ZERO, &[], &[]).is_err());

            // Make some changes to r1, r2.
            let pre1 = Arc::new(r1.clone().into_sealed_committed());
            let pre2 = Arc::new(r2.clone().into_sealed_committed());
            r1.add_ava(Attribute::TestAttr, Value::from("modified"));
            r2.add_ava(Attribute::TestAttr, Value::from("modified"));

            // Now ... cheat.

            let vr1 = r1.into_sealed_committed();
            let vr2 = r2.into_sealed_committed();

            // Modify single
            assert!(be
                .modify(&CID_ZERO, &[pre1], std::slice::from_ref(&vr1))
                .is_ok());
            // Assert no other changes
            assert!(entry_attr_pres!(be, vr1, Attribute::TestAttr));
            assert!(!entry_attr_pres!(be, vr2, Attribute::TestAttr));

            // Modify both
            assert!(be
                .modify(
                    &CID_ZERO,
                    &[Arc::new(vr1.clone()), pre2],
                    &[vr1.clone(), vr2.clone()]
                )
                .is_ok());

            assert!(entry_attr_pres!(be, vr1, Attribute::TestAttr));
            assert!(entry_attr_pres!(be, vr2, Attribute::TestAttr));
        });
    }

    #[test]
    fn test_be_simple_delete() {
        run_test!(|be: &mut BackendWriteTransaction| {
            trace!("Simple Delete");
            let lims = Limits::unlimited();

            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::UserId, Value::from("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::UserId, Value::from("alice"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"),
            );

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava(Attribute::UserId, Value::from("lucy"));
            e3.add_ava(
                Attribute::Uuid,
                Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"),
            );

            let ve1 = e1.clone().into_sealed_new();
            let ve2 = e2.clone().into_sealed_new();
            let ve3 = e3.clone().into_sealed_new();

            assert!(be.create(&CID_ZERO, vec![ve1, ve2, ve3]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));
            assert!(entry_exists!(be, e3));

            // You need to now retrieve the entries back out to get the entry id's
            let mut results = be
                .search(&lims, &filter_resolved!(f_pres(Attribute::UserId)))
                .expect("Failed to search");

            // Get these out to usable entries.
            let r1 = results.remove(0);
            let r2 = results.remove(0);
            let r3 = results.remove(0);

            // Deletes nothing, all entries are live.
            assert!(matches!(be.reap_tombstones(&CID_ADV, &CID_ZERO), Ok(0)));

            // Put them into the tombstone state, and write that down.
            // This sets up the RUV with the changes.
            let r1_ts = r1.to_tombstone(CID_ONE.clone()).into_sealed_committed();

            assert!(be
                .modify(&CID_ONE, &[r1], std::slice::from_ref(&r1_ts))
                .is_ok());

            let r2_ts = r2.to_tombstone(CID_TWO.clone()).into_sealed_committed();
            let r3_ts = r3.to_tombstone(CID_TWO.clone()).into_sealed_committed();

            assert!(be
                .modify(&CID_TWO, &[r2, r3], &[r2_ts.clone(), r3_ts.clone()])
                .is_ok());

            // The entry are now tombstones, but is still in the ruv. This is because we
            // targeted CID_ZERO, not ONE.
            assert!(matches!(be.reap_tombstones(&CID_ADV, &CID_ZERO), Ok(0)));

            assert!(entry_exists!(be, r1_ts));
            assert!(entry_exists!(be, r2_ts));
            assert!(entry_exists!(be, r3_ts));

            assert!(matches!(be.reap_tombstones(&CID_ADV, &CID_ONE), Ok(0)));

            assert!(entry_exists!(be, r1_ts));
            assert!(entry_exists!(be, r2_ts));
            assert!(entry_exists!(be, r3_ts));

            assert!(matches!(be.reap_tombstones(&CID_ADV, &CID_TWO), Ok(1)));

            assert!(!entry_exists!(be, r1_ts));
            assert!(entry_exists!(be, r2_ts));
            assert!(entry_exists!(be, r3_ts));

            assert!(matches!(be.reap_tombstones(&CID_ADV, &CID_THREE), Ok(2)));

            assert!(!entry_exists!(be, r1_ts));
            assert!(!entry_exists!(be, r2_ts));
            assert!(!entry_exists!(be, r3_ts));

            // Nothing left
            assert!(matches!(be.reap_tombstones(&CID_ADV, &CID_THREE), Ok(0)));

            assert!(!entry_exists!(be, r1_ts));
            assert!(!entry_exists!(be, r2_ts));
            assert!(!entry_exists!(be, r3_ts));
        });
    }

    #[test]
    fn test_be_backup_restore() {
        let db_backup_file_name =
            Path::new(option_env!("OUT_DIR").unwrap_or("/tmp")).join(".backup_test.json.gz");
        eprintln!(" ⚠️   {}", db_backup_file_name.display());
        run_test!(|be: &mut BackendWriteTransaction| {
            // Important! Need db metadata setup!
            be.reset_db_s_uuid().unwrap();
            be.reset_db_d_uuid().unwrap();
            be.set_db_ts_max(Duration::from_secs(1)).unwrap();

            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::UserId, Value::from("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::UserId, Value::from("alice"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"),
            );

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava(Attribute::UserId, Value::from("lucy"));
            e3.add_ava(
                Attribute::Uuid,
                Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"),
            );

            let ve1 = e1.clone().into_sealed_new();
            let ve2 = e2.clone().into_sealed_new();
            let ve3 = e3.clone().into_sealed_new();

            assert!(be.create(&CID_ZERO, vec![ve1, ve2, ve3]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));
            assert!(entry_exists!(be, e3));

            let result = fs::remove_file(&db_backup_file_name);

            if let Err(e) = result {
                // if the error is the file is not found, that's what we want so continue,
                // otherwise return the error
                if e.kind() == std::io::ErrorKind::NotFound {}
            }
            be.backup(&db_backup_file_name, BackupCompression::Gzip)
                .expect("Backup failed!");
            be.restore(&db_backup_file_name).expect("Restore failed!");

            assert!(be.verify().is_empty());
        });
    }

    #[test]
    fn test_be_backup_restore_tampered() {
        let db_backup_file_name =
            Path::new(option_env!("OUT_DIR").unwrap_or("/tmp")).join(".backup2_test.json");
        eprintln!(" ⚠️   {}", db_backup_file_name.display());
        run_test!(|be: &mut BackendWriteTransaction| {
            // Important! Need db metadata setup!
            be.reset_db_s_uuid().unwrap();
            be.reset_db_d_uuid().unwrap();
            be.set_db_ts_max(Duration::from_secs(1)).unwrap();
            // First create some entries (3?)
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::UserId, Value::from("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::UserId, Value::from("alice"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("4b6228ab-1dbe-42a4-a9f5-f6368222438e"),
            );

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava(Attribute::UserId, Value::from("lucy"));
            e3.add_ava(
                Attribute::Uuid,
                Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"),
            );

            let ve1 = e1.clone().into_sealed_new();
            let ve2 = e2.clone().into_sealed_new();
            let ve3 = e3.clone().into_sealed_new();

            assert!(be.create(&CID_ZERO, vec![ve1, ve2, ve3]).is_ok());
            assert!(entry_exists!(be, e1));
            assert!(entry_exists!(be, e2));
            assert!(entry_exists!(be, e3));

            let result = fs::remove_file(&db_backup_file_name);

            if let Err(e) = result {
                // if the error is the file is not found, that's what we want so continue,
                // otherwise return the error
                if e.kind() == std::io::ErrorKind::NotFound {}
            }

            be.backup(&db_backup_file_name, BackupCompression::NoCompression)
                .expect("Backup failed!");

            // Now here, we need to tamper with the file.
            let serialized_string = fs::read_to_string(&db_backup_file_name).unwrap();
            trace!(?serialized_string);
            let mut dbbak: DbBackup = serde_json::from_str(&serialized_string).unwrap();

            match &mut dbbak {
                DbBackup::V5 {
                    version: _,
                    db_s_uuid: _,
                    db_d_uuid: _,
                    db_ts_max: _,
                    keyhandles: _,
                    repl_meta: _,
                    entries,
                } => {
                    let _ = entries.pop();
                }
                _ => {
                    // We no longer use these format versions!
                    unreachable!()
                }
            };

            let serialized_entries_str = serde_json::to_string_pretty(&dbbak).unwrap();
            fs::write(&db_backup_file_name, serialized_entries_str).unwrap();

            be.restore(&db_backup_file_name).expect("Restore failed!");

            assert!(be.verify().is_empty());
        });
    }

    #[test]
    fn test_be_sid_generation_and_reset() {
        run_test!(|be: &mut BackendWriteTransaction| {
            let sid1 = be.get_db_s_uuid().unwrap();
            let sid2 = be.get_db_s_uuid().unwrap();
            assert_eq!(sid1, sid2);
            let sid3 = be.reset_db_s_uuid().unwrap();
            assert!(sid1 != sid3);
            let sid4 = be.get_db_s_uuid().unwrap();
            assert_eq!(sid3, sid4);
        });
    }

    #[test]
    fn test_be_reindex_empty() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Add some test data?
            let missing = be.missing_idxs().unwrap();
            assert_eq!(missing.len(), 7);
            assert!(be.reindex(false).is_ok());
            let missing = be.missing_idxs().unwrap();
            debug!("{:?}", missing);
            assert!(missing.is_empty());
        });
    }

    #[test]
    fn test_be_reindex_data() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Add some test data?
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::Name, Value::new_iname("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            let e1 = e1.into_sealed_new();

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::Name, Value::new_iname("claire"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f"),
            );
            let e2 = e2.into_sealed_new();

            be.create(&CID_ZERO, vec![e1, e2]).unwrap();

            // purge indexes
            be.danger_purge_idxs().unwrap();
            // Check they are gone
            let missing = be.missing_idxs().unwrap();
            assert_eq!(missing.len(), 7);
            assert!(be.reindex(false).is_ok());
            let missing = be.missing_idxs().unwrap();
            debug!("{:?}", missing);
            assert!(missing.is_empty());
            // check name and uuid ids on eq, sub, pres

            idl_state!(
                be,
                Attribute::Name,
                IndexType::Equality,
                "william",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Name,
                IndexType::Equality,
                "claire",
                Some(vec![2])
            );

            for sub in [
                "w", "m", "wi", "il", "ll", "li", "ia", "am", "wil", "ill", "lli", "lia", "iam",
            ] {
                idl_state!(
                    be,
                    Attribute::Name,
                    IndexType::SubString,
                    sub,
                    Some(vec![1])
                );
            }

            for sub in [
                "c", "r", "e", "cl", "la", "ai", "ir", "re", "cla", "lai", "air", "ire",
            ] {
                idl_state!(
                    be,
                    Attribute::Name,
                    IndexType::SubString,
                    sub,
                    Some(vec![2])
                );
            }

            for sub in ["i", "a", "l"] {
                idl_state!(
                    be,
                    Attribute::Name,
                    IndexType::SubString,
                    sub,
                    Some(vec![1, 2])
                );
            }

            idl_state!(
                be,
                Attribute::Name,
                IndexType::Presence,
                "_",
                Some(vec![1, 2])
            );

            idl_state!(
                be,
                Attribute::Uuid,
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Uuid,
                IndexType::Equality,
                "bd651620-00dd-426b-aaa0-4494f7b7906f",
                Some(vec![2])
            );

            idl_state!(
                be,
                Attribute::Uuid,
                IndexType::Presence,
                "_",
                Some(vec![1, 2])
            );

            // Show what happens with empty

            idl_state!(
                be,
                Attribute::Name,
                IndexType::Equality,
                "not-exist",
                Some(Vec::with_capacity(0))
            );

            idl_state!(
                be,
                Attribute::Uuid,
                IndexType::Equality,
                "fake-0079-4b8c-8a56-593b22aa44d1",
                Some(Vec::with_capacity(0))
            );

            let uuid_p_idl = be
                .load_test_idl(&Attribute::from("not_indexed"), IndexType::Presence, "_")
                .unwrap(); // unwrap the result
            assert_eq!(uuid_p_idl, None);

            // Check name2uuid
            let claire_uuid = uuid!("bd651620-00dd-426b-aaa0-4494f7b7906f");
            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");

            assert_eq!(be.name2uuid("claire"), Ok(Some(claire_uuid)));
            assert_eq!(be.name2uuid("william"), Ok(Some(william_uuid)));
            assert_eq!(
                be.name2uuid("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
                Ok(None)
            );
            // check uuid2spn
            assert_eq!(
                be.uuid2spn(claire_uuid),
                Ok(Some(Value::new_iname("claire")))
            );
            assert_eq!(
                be.uuid2spn(william_uuid),
                Ok(Some(Value::new_iname("william")))
            );
            // check uuid2rdn
            assert_eq!(
                be.uuid2rdn(claire_uuid),
                Ok(Some("name=claire".to_string()))
            );
            assert_eq!(
                be.uuid2rdn(william_uuid),
                Ok(Some("name=william".to_string()))
            );
        });
    }

    #[test]
    fn test_be_index_create_delete_simple() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // First, setup our index tables!
            assert!(be.reindex(false).is_ok());
            // Test that on entry create, the indexes are made correctly.
            // this is a similar case to reindex.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::Name, Value::from("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            let e1 = e1.into_sealed_new();

            let rset = be.create(&CID_ZERO, vec![e1]).unwrap();
            let mut rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            let e1 = rset.pop().unwrap();

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Equality,
                "william",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Presence,
                "_",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Presence,
                "_",
                Some(vec![1])
            );

            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            assert_eq!(be.name2uuid("william"), Ok(Some(william_uuid)));
            assert_eq!(be.uuid2spn(william_uuid), Ok(Some(Value::from("william"))));
            assert_eq!(
                be.uuid2rdn(william_uuid),
                Ok(Some("name=william".to_string()))
            );

            // == Now we reap_tombstones, and assert we removed the items.
            let e1_ts = e1.to_tombstone(CID_ONE.clone()).into_sealed_committed();
            assert!(be.modify(&CID_ONE, &[e1], &[e1_ts]).is_ok());
            be.reap_tombstones(&CID_ADV, &CID_TWO).unwrap();

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Equality,
                "william",
                Some(Vec::with_capacity(0))
            );

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Presence,
                "_",
                Some(Vec::with_capacity(0))
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(Vec::with_capacity(0))
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Presence,
                "_",
                Some(Vec::with_capacity(0))
            );

            assert_eq!(be.name2uuid("william"), Ok(None));
            assert_eq!(be.uuid2spn(william_uuid), Ok(None));
            assert_eq!(be.uuid2rdn(william_uuid), Ok(None));
        })
    }

    #[test]
    fn test_be_index_create_delete_multi() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // delete multiple entries at a time, without deleting others
            // First, setup our index tables!
            assert!(be.reindex(false).is_ok());
            // Test that on entry create, the indexes are made correctly.
            // this is a similar case to reindex.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::Name, Value::new_iname("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            let e1 = e1.into_sealed_new();

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::Name, Value::new_iname("claire"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("bd651620-00dd-426b-aaa0-4494f7b7906f"),
            );
            let e2 = e2.into_sealed_new();

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava(Attribute::UserId, Value::new_iname("lucy"));
            e3.add_ava(
                Attribute::Uuid,
                Value::from("7b23c99d-c06b-4a9a-a958-3afa56383e1d"),
            );
            let e3 = e3.into_sealed_new();

            let mut rset = be.create(&CID_ZERO, vec![e1, e2, e3]).unwrap();
            rset.remove(1);
            let mut rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            let e1 = rset.pop().unwrap();
            let e3 = rset.pop().unwrap();

            // Now remove e1, e3.
            let e1_ts = e1.to_tombstone(CID_ONE.clone()).into_sealed_committed();
            let e3_ts = e3.to_tombstone(CID_ONE.clone()).into_sealed_committed();
            assert!(be.modify(&CID_ONE, &[e1, e3], &[e1_ts, e3_ts]).is_ok());
            be.reap_tombstones(&CID_ADV, &CID_TWO).unwrap();

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Equality,
                "claire",
                Some(vec![2])
            );

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Presence,
                "_",
                Some(vec![2])
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Equality,
                "bd651620-00dd-426b-aaa0-4494f7b7906f",
                Some(vec![2])
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Presence,
                "_",
                Some(vec![2])
            );

            let claire_uuid = uuid!("bd651620-00dd-426b-aaa0-4494f7b7906f");
            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            let lucy_uuid = uuid!("7b23c99d-c06b-4a9a-a958-3afa56383e1d");

            assert_eq!(be.name2uuid("claire"), Ok(Some(claire_uuid)));
            let x = be.uuid2spn(claire_uuid);
            trace!(?x);
            assert_eq!(
                be.uuid2spn(claire_uuid),
                Ok(Some(Value::new_iname("claire")))
            );
            assert_eq!(
                be.uuid2rdn(claire_uuid),
                Ok(Some("name=claire".to_string()))
            );

            assert_eq!(be.name2uuid("william"), Ok(None));
            assert_eq!(be.uuid2spn(william_uuid), Ok(None));
            assert_eq!(be.uuid2rdn(william_uuid), Ok(None));

            assert_eq!(be.name2uuid("lucy"), Ok(None));
            assert_eq!(be.uuid2spn(lucy_uuid), Ok(None));
            assert_eq!(be.uuid2rdn(lucy_uuid), Ok(None));
        })
    }

    #[test]
    fn test_be_index_modify_simple() {
        run_test!(|be: &mut BackendWriteTransaction| {
            assert!(be.reindex(false).is_ok());
            // modify with one type, ensuring we clean the indexes behind
            // us. For the test to be "accurate" we must add one attr, remove one attr
            // and change one attr.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::Name, Value::new_iname("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            e1.add_ava(Attribute::TestAttr, Value::from("test"));
            let e1 = e1.into_sealed_new();

            let rset = be.create(&CID_ZERO, vec![e1]).unwrap();
            let rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            // Now, alter the new entry.
            let mut ce1 = rset[0].as_ref().clone().into_invalid();
            // add something.
            ce1.add_ava(Attribute::TestNumber, Value::from("test"));
            // remove something.
            ce1.purge_ava(Attribute::TestAttr);
            // mod something.
            ce1.purge_ava(Attribute::Name);
            ce1.add_ava(Attribute::Name, Value::new_iname("claire"));

            let ce1 = ce1.into_sealed_committed();

            be.modify(&CID_ZERO, &rset, &[ce1]).unwrap();

            // Now check the idls
            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Equality,
                "claire",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Presence,
                "_",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::TestNumber.as_ref(),
                IndexType::Equality,
                "test",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::TestAttr,
                IndexType::Equality,
                "test",
                Some(vec![])
            );

            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            assert_eq!(be.name2uuid("william"), Ok(None));
            assert_eq!(be.name2uuid("claire"), Ok(Some(william_uuid)));
            assert_eq!(
                be.uuid2spn(william_uuid),
                Ok(Some(Value::new_iname("claire")))
            );
            assert_eq!(
                be.uuid2rdn(william_uuid),
                Ok(Some("name=claire".to_string()))
            );
        })
    }

    #[test]
    fn test_be_index_modify_rename() {
        run_test!(|be: &mut BackendWriteTransaction| {
            assert!(be.reindex(false).is_ok());
            // test when we change name AND uuid
            // This will be needing to be correct for conflicts when we add
            // replication support!
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::Name, Value::new_iname("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            let e1 = e1.into_sealed_new();

            let rset = be.create(&CID_ZERO, vec![e1]).unwrap();
            let rset: Vec<_> = rset.into_iter().map(Arc::new).collect();
            // Now, alter the new entry.
            let mut ce1 = rset[0].as_ref().clone().into_invalid();
            ce1.purge_ava(Attribute::Name);
            ce1.purge_ava(Attribute::Uuid);
            ce1.add_ava(Attribute::Name, Value::new_iname("claire"));
            ce1.add_ava(
                Attribute::Uuid,
                Value::from("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8"),
            );
            let ce1 = ce1.into_sealed_committed();

            be.modify(&CID_ZERO, &rset, &[ce1]).unwrap();

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Equality,
                "claire",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Equality,
                "04091a7a-6ce4-42d2-abf5-c2ce244ac9e8",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Presence,
                "_",
                Some(vec![1])
            );
            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Presence,
                "_",
                Some(vec![1])
            );

            idl_state!(
                be,
                Attribute::Uuid.as_ref(),
                IndexType::Equality,
                "db237e8a-0079-4b8c-8a56-593b22aa44d1",
                Some(Vec::with_capacity(0))
            );
            idl_state!(
                be,
                Attribute::Name.as_ref(),
                IndexType::Equality,
                "william",
                Some(Vec::with_capacity(0))
            );

            let claire_uuid = uuid!("04091a7a-6ce4-42d2-abf5-c2ce244ac9e8");
            let william_uuid = uuid!("db237e8a-0079-4b8c-8a56-593b22aa44d1");
            assert_eq!(be.name2uuid("william"), Ok(None));
            assert_eq!(be.name2uuid("claire"), Ok(Some(claire_uuid)));
            assert_eq!(be.uuid2spn(william_uuid), Ok(None));
            assert_eq!(be.uuid2rdn(william_uuid), Ok(None));
            assert_eq!(
                be.uuid2spn(claire_uuid),
                Ok(Some(Value::new_iname("claire")))
            );
            assert_eq!(
                be.uuid2rdn(claire_uuid),
                Ok(Some("name=claire".to_string()))
            );
        })
    }

    #[test]
    fn test_be_index_search_simple() {
        run_test!(|be: &mut BackendWriteTransaction| {
            assert!(be.reindex(false).is_ok());

            // Create a test entry with some indexed / unindexed values.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::Name, Value::new_iname("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            e1.add_ava(Attribute::NoIndex, Value::from("william"));
            e1.add_ava(Attribute::OtherNoIndex, Value::from("william"));
            let e1 = e1.into_sealed_new();

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::Name, Value::new_iname("claire"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d2"),
            );
            let e2 = e2.into_sealed_new();

            let _rset = be.create(&CID_ZERO, vec![e1, e2]).unwrap();
            // Test fully unindexed
            let f_un =
                filter_resolved!(f_eq(Attribute::NoIndex, PartialValue::new_utf8s("william")));

            let (r, _plan) = be.filter2idl(f_un.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            // Test that a fully indexed search works
            let feq = filter_resolved!(f_eq(Attribute::Name, PartialValue::new_utf8s("william")));

            let (r, _plan) = be.filter2idl(feq.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }

            // Test and/or
            //   full index and
            let f_in_and = filter_resolved!(f_and!([
                f_eq(Attribute::Name, PartialValue::new_utf8s("william")),
                f_eq(
                    Attribute::Uuid,
                    PartialValue::new_utf8s("db237e8a-0079-4b8c-8a56-593b22aa44d1")
                )
            ]));

            let (r, _plan) = be.filter2idl(f_in_and.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }

            //   partial index and
            let f_p1 = filter_resolved!(f_and!([
                f_eq(Attribute::Name, PartialValue::new_utf8s("william")),
                f_eq(Attribute::NoIndex, PartialValue::new_utf8s("william"))
            ]));

            let f_p2 = filter_resolved!(f_and!([
                f_eq(Attribute::Name, PartialValue::new_utf8s("william")),
                f_eq(Attribute::NoIndex, PartialValue::new_utf8s("william"))
            ]));

            let (r, _plan) = be.filter2idl(f_p1.to_inner(), 0).unwrap();
            match r {
                IdList::Partial(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => unreachable!(),
            }

            let (r, _plan) = be.filter2idl(f_p2.to_inner(), 0).unwrap();
            match r {
                IdList::Partial(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => unreachable!(),
            }

            // Substrings are always partial
            let f_p3 = filter_resolved!(f_sub(Attribute::Name, PartialValue::new_utf8s("wil")));

            let (r, plan) = be.filter2idl(f_p3.to_inner(), 0).unwrap();
            trace!(?r, ?plan);
            match r {
                IdList::Partial(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => unreachable!(),
            }

            //   no index and
            let f_no_and = filter_resolved!(f_and!([
                f_eq(Attribute::NoIndex, PartialValue::new_utf8s("william")),
                f_eq(Attribute::OtherNoIndex, PartialValue::new_utf8s("william"))
            ]));

            let (r, _plan) = be.filter2idl(f_no_and.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            //   full index or
            let f_in_or = filter_resolved!(f_or!([f_eq(
                Attribute::Name,
                PartialValue::new_utf8s("william")
            )]));

            let (r, _plan) = be.filter2idl(f_in_or.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }
            //   partial (aka allids) or
            let f_un_or = filter_resolved!(f_or!([f_eq(
                Attribute::NoIndex,
                PartialValue::new_utf8s("william")
            )]));

            let (r, _plan) = be.filter2idl(f_un_or.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            // Test root andnot
            let f_r_andnot = filter_resolved!(f_andnot(f_eq(
                Attribute::Name,
                PartialValue::new_utf8s("william")
            )));

            let (r, _plan) = be.filter2idl(f_r_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(Vec::with_capacity(0)));
                }
                _ => {
                    panic!("");
                }
            }

            // test andnot as only in and
            let f_and_andnot = filter_resolved!(f_and!([f_andnot(f_eq(
                Attribute::Name,
                PartialValue::new_utf8s("william")
            ))]));

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(Vec::with_capacity(0)));
                }
                _ => {
                    panic!("");
                }
            }
            // test andnot as only in or
            let f_or_andnot = filter_resolved!(f_or!([f_andnot(f_eq(
                Attribute::Name,
                PartialValue::new_utf8s("william")
            ))]));

            let (r, _plan) = be.filter2idl(f_or_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(Vec::with_capacity(0)));
                }
                _ => {
                    panic!("");
                }
            }

            // test andnot in and (first) with name
            let f_and_andnot = filter_resolved!(f_and!([
                f_andnot(f_eq(Attribute::Name, PartialValue::new_utf8s("claire"))),
                f_pres(Attribute::Name)
            ]));

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    debug!("{:?}", idl);
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }
            // test andnot in and (last) with name
            let f_and_andnot = filter_resolved!(f_and!([
                f_pres(Attribute::Name),
                f_andnot(f_eq(Attribute::Name, PartialValue::new_utf8s("claire")))
            ]));

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![1]));
                }
                _ => {
                    panic!("");
                }
            }
            // test andnot in and (first) with no-index
            let f_and_andnot = filter_resolved!(f_and!([
                f_andnot(f_eq(Attribute::Name, PartialValue::new_utf8s("claire"))),
                f_pres(Attribute::NoIndex)
            ]));

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }
            // test andnot in and (last) with no-index
            let f_and_andnot = filter_resolved!(f_and!([
                f_pres(Attribute::NoIndex),
                f_andnot(f_eq(Attribute::Name, PartialValue::new_utf8s("claire")))
            ]));

            let (r, _plan) = be.filter2idl(f_and_andnot.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }

            //   empty or
            let f_e_or = filter_resolved!(f_or!([]));

            let (r, _plan) = be.filter2idl(f_e_or.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![]));
                }
                _ => {
                    panic!("");
                }
            }

            let f_e_and = filter_resolved!(f_and!([]));

            let (r, _plan) = be.filter2idl(f_e_and.to_inner(), 0).unwrap();
            match r {
                IdList::Indexed(idl) => {
                    assert_eq!(idl, IDLBitRange::from_iter(vec![]));
                }
                _ => {
                    panic!("");
                }
            }
        })
    }

    #[test]
    fn test_be_index_search_missing() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Test where the index is in schema but not created (purge idxs)
            // should fall back to an empty set because we can't satisfy the term
            be.danger_purge_idxs().unwrap();
            debug!("{:?}", be.missing_idxs().unwrap());
            let f_eq = filter_resolved!(f_eq(Attribute::Name, PartialValue::new_utf8s("william")));

            let (r, _plan) = be.filter2idl(f_eq.to_inner(), 0).unwrap();
            match r {
                IdList::AllIds => {}
                _ => {
                    panic!("");
                }
            }
        })
    }

    #[test]
    fn test_be_index_slope_generation() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // Create some test entry with some indexed / unindexed values.
            let mut e1: Entry<EntryInit, EntryNew> = Entry::new();
            e1.add_ava(Attribute::Name, Value::new_iname("william"));
            e1.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            e1.add_ava(Attribute::TestAttr, Value::from("dupe"));
            e1.add_ava(Attribute::TestNumber, Value::from("1"));
            let e1 = e1.into_sealed_new();

            let mut e2: Entry<EntryInit, EntryNew> = Entry::new();
            e2.add_ava(Attribute::Name, Value::new_iname("claire"));
            e2.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d2"),
            );
            e2.add_ava(Attribute::TestAttr, Value::from("dupe"));
            e2.add_ava(Attribute::TestNumber, Value::from("1"));
            let e2 = e2.into_sealed_new();

            let mut e3: Entry<EntryInit, EntryNew> = Entry::new();
            e3.add_ava(Attribute::Name, Value::new_iname("benny"));
            e3.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d3"),
            );
            e3.add_ava(Attribute::TestAttr, Value::from("dupe"));
            e3.add_ava(Attribute::TestNumber, Value::from("2"));
            let e3 = e3.into_sealed_new();

            let _rset = be.create(&CID_ZERO, vec![e1, e2, e3]).unwrap();

            // If the slopes haven't been generated yet, there are some hardcoded values
            // that we can use instead. They aren't generated until a first re-index.
            assert!(!be.is_idx_slopeyness_generated().unwrap());

            let ta_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::TestAttr, IndexType::Equality))
                .unwrap();
            assert_eq!(ta_eq_slope, 45);

            let tb_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::TestNumber, IndexType::Equality))
                .unwrap();
            assert_eq!(tb_eq_slope, 45);

            let name_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Name, IndexType::Equality))
                .unwrap();
            assert_eq!(name_eq_slope, 1);
            let uuid_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Uuid, IndexType::Equality))
                .unwrap();
            assert_eq!(uuid_eq_slope, 1);

            let name_pres_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Name, IndexType::Presence))
                .unwrap();
            assert_eq!(name_pres_slope, 90);
            let uuid_pres_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Uuid, IndexType::Presence))
                .unwrap();
            assert_eq!(uuid_pres_slope, 90);
            // Check the slopes are what we expect for hardcoded values.

            // Now check slope generation for the values. Today these are calculated
            // at reindex time, so we now perform the re-index.
            assert!(be.reindex(false).is_ok());
            assert!(be.is_idx_slopeyness_generated().unwrap());

            let ta_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::TestAttr, IndexType::Equality))
                .unwrap();
            assert_eq!(ta_eq_slope, 200);

            let tb_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::TestNumber, IndexType::Equality))
                .unwrap();
            assert_eq!(tb_eq_slope, 133);

            let name_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Name, IndexType::Equality))
                .unwrap();
            assert_eq!(name_eq_slope, 51);
            let uuid_eq_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Uuid, IndexType::Equality))
                .unwrap();
            assert_eq!(uuid_eq_slope, 51);

            let name_pres_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Name, IndexType::Presence))
                .unwrap();
            assert_eq!(name_pres_slope, 200);
            let uuid_pres_slope = be
                .get_idx_slope(&IdxKey::new(Attribute::Uuid, IndexType::Presence))
                .unwrap();
            assert_eq!(uuid_pres_slope, 200);
        })
    }

    #[test]
    fn test_be_limits_allids() {
        run_test!(|be: &mut BackendWriteTransaction| {
            let mut lim_allow_allids = Limits::unlimited();
            lim_allow_allids.unindexed_allow = true;

            let mut lim_deny_allids = Limits::unlimited();
            lim_deny_allids.unindexed_allow = false;

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava(Attribute::UserId, Value::from("william"));
            e.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            e.add_ava(Attribute::NonExist, Value::from("x"));
            let e = e.into_sealed_new();
            let single_result = be.create(&CID_ZERO, vec![e.clone()]);

            assert!(single_result.is_ok());
            let filt = e
                .filter_from_attrs(&[Attribute::NonExist])
                .expect("failed to generate filter")
                .into_valid_resolved();
            // check allow on allids
            let res = be.search(&lim_allow_allids, &filt);
            assert!(res.is_ok());
            let res = be.exists(&lim_allow_allids, &filt);
            assert!(res.is_ok());

            // check deny on allids
            let res = be.search(&lim_deny_allids, &filt);
            assert_eq!(res, Err(OperationError::ResourceLimit));
            let res = be.exists(&lim_deny_allids, &filt);
            assert_eq!(res, Err(OperationError::ResourceLimit));
        })
    }

    #[test]
    fn test_be_limits_results_max() {
        run_test!(|be: &mut BackendWriteTransaction| {
            let mut lim_allow = Limits::unlimited();
            lim_allow.search_max_results = usize::MAX;

            let mut lim_deny = Limits::unlimited();
            lim_deny.search_max_results = 0;

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava(Attribute::UserId, Value::from("william"));
            e.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            e.add_ava(Attribute::NonExist, Value::from("x"));
            let e = e.into_sealed_new();
            let single_result = be.create(&CID_ZERO, vec![e.clone()]);
            assert!(single_result.is_ok());

            let filt = e
                .filter_from_attrs(&[Attribute::NonExist])
                .expect("failed to generate filter")
                .into_valid_resolved();

            // --> This is the all ids path (unindexed)
            // check allow on entry max
            let res = be.search(&lim_allow, &filt);
            assert!(res.is_ok());
            let res = be.exists(&lim_allow, &filt);
            assert!(res.is_ok());

            // check deny on entry max
            let res = be.search(&lim_deny, &filt);
            assert_eq!(res, Err(OperationError::ResourceLimit));
            // we don't limit on exists because we never load the entries.
            let res = be.exists(&lim_deny, &filt);
            assert!(res.is_ok());

            // --> This will shortcut due to indexing.
            assert!(be.reindex(false).is_ok());
            let res = be.search(&lim_deny, &filt);
            assert_eq!(res, Err(OperationError::ResourceLimit));
            // we don't limit on exists because we never load the entries.
            let res = be.exists(&lim_deny, &filt);
            assert!(res.is_ok());
        })
    }

    #[test]
    fn test_be_limits_partial_filter() {
        run_test!(|be: &mut BackendWriteTransaction| {
            // This relies on how we do partials, so it could be a bit sensitive.
            // A partial is generated after an allids + indexed in a single and
            // as we require both conditions to exist. Allids comes from unindexed
            // terms. we need to ensure we don't hit partial threshold too.
            //
            // This means we need an and query where the first term is allids
            // and the second is indexed, but without the filter shortcutting.
            //
            // To achieve this we need a monstrously evil query.
            //
            let mut lim_allow = Limits::unlimited();
            lim_allow.search_max_filter_test = usize::MAX;

            let mut lim_deny = Limits::unlimited();
            lim_deny.search_max_filter_test = 0;

            let mut e: Entry<EntryInit, EntryNew> = Entry::new();
            e.add_ava(Attribute::Name, Value::new_iname("william"));
            e.add_ava(
                Attribute::Uuid,
                Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
            );
            e.add_ava(Attribute::NonExist, Value::from("x"));
            e.add_ava(Attribute::NonExist, Value::from("y"));
            let e = e.into_sealed_new();
            let single_result = be.create(&CID_ZERO, vec![e]);
            assert!(single_result.is_ok());

            // Reindex so we have things in place for our query
            assert!(be.reindex(false).is_ok());

            // 🚨 This is evil!
            // The and allows us to hit "allids + indexed -> partial".
            // the or terms prevent re-arrangement. They can't be folded or dead
            // term elimed either.
            //
            // This means the f_or nonexist will become allids and the second will be indexed
            // due to f_eq userid in both with the result of william.
            //
            // This creates a partial, and because it's the first iteration in the loop, this
            // doesn't encounter partial threshold testing.
            let filt = filter_resolved!(f_and!([
                f_or!([
                    f_eq(Attribute::NonExist, PartialValue::new_utf8s("x")),
                    f_eq(Attribute::NonExist, PartialValue::new_utf8s("y"))
                ]),
                f_or!([
                    f_eq(Attribute::Name, PartialValue::new_utf8s("claire")),
                    f_eq(Attribute::Name, PartialValue::new_utf8s("william"))
                ]),
            ]));

            let res = be.search(&lim_allow, &filt);
            assert!(res.is_ok());
            let res = be.exists(&lim_allow, &filt);
            assert!(res.is_ok());

            // check deny on entry max
            let res = be.search(&lim_deny, &filt);
            assert_eq!(res, Err(OperationError::ResourceLimit));
            // we don't limit on exists because we never load the entries.
            let res = be.exists(&lim_deny, &filt);
            assert_eq!(res, Err(OperationError::ResourceLimit));
        })
    }

    #[test]
    fn test_be_multiple_create() {
        sketching::test_init();

        // This is a demo idxmeta, purely for testing.
        let idxmeta = vec![IdxKey {
            attr: Attribute::Uuid,
            itype: IndexType::Equality,
        }];

        let be_a = Backend::new(BackendConfig::new_test("main"), idxmeta.clone(), false)
            .expect("Failed to setup backend");

        let be_b = Backend::new(BackendConfig::new_test("db_2"), idxmeta, false)
            .expect("Failed to setup backend");

        let mut be_a_txn = be_a.write().unwrap();
        let mut be_b_txn = be_b.write().unwrap();

        assert!(be_a_txn.get_db_s_uuid() != be_b_txn.get_db_s_uuid());

        // Create into A
        let mut e: Entry<EntryInit, EntryNew> = Entry::new();
        e.add_ava(Attribute::UserId, Value::from("william"));
        e.add_ava(
            Attribute::Uuid,
            Value::from("db237e8a-0079-4b8c-8a56-593b22aa44d1"),
        );
        let e = e.into_sealed_new();

        let single_result = be_a_txn.create(&CID_ZERO, vec![e]);

        assert!(single_result.is_ok());

        // Assert it's in A but not B.
        let filt = filter_resolved!(f_eq(Attribute::UserId, PartialValue::new_utf8s("william")));

        let lims = Limits::unlimited();

        let r = be_a_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").len() == 1);

        let r = be_b_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").is_empty());

        // Create into B
        let mut e: Entry<EntryInit, EntryNew> = Entry::new();
        e.add_ava(Attribute::UserId, Value::from("claire"));
        e.add_ava(
            Attribute::Uuid,
            Value::from("0c680959-0944-47d6-9dea-53304d124266"),
        );
        let e = e.into_sealed_new();

        let single_result = be_b_txn.create(&CID_ZERO, vec![e]);

        assert!(single_result.is_ok());

        // Assert it's in B but not A
        let filt = filter_resolved!(f_eq(Attribute::UserId, PartialValue::new_utf8s("claire")));

        let lims = Limits::unlimited();

        let r = be_a_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").is_empty());

        let r = be_b_txn.search(&lims, &filt);
        assert!(r.expect("Search failed!").len() == 1);
    }
}