fix: store IndexReader once; sanitize all tantivy query specials

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-05-18 16:51:48 +02:00
parent 6dd1c17306
commit a896c5bbde
+15 -8
View File
@@ -7,6 +7,7 @@ use tantivy::{doc, Index, TantivyDocument};
pub struct LexicalIndex { pub struct LexicalIndex {
index: Index, index: Index,
reader: tantivy::IndexReader,
text_field: tantivy::schema::Field, text_field: tantivy::schema::Field,
idx_field: tantivy::schema::Field, idx_field: tantivy::schema::Field,
} }
@@ -31,8 +32,12 @@ impl LexicalIndex {
} }
w.commit() w.commit()
.map_err(|e| AppError::Io(format!("tantivy commit: {e}")))?; .map_err(|e| AppError::Io(format!("tantivy commit: {e}")))?;
let reader = index
.reader()
.map_err(|e| AppError::Io(format!("tantivy reader: {e}")))?;
Ok(Self { Ok(Self {
index, index,
reader,
text_field, text_field,
idx_field, idx_field,
}) })
@@ -40,21 +45,23 @@ impl LexicalIndex {
/// Returns (entry_index, bm25_score) descending. /// Returns (entry_index, bm25_score) descending.
pub fn search(&self, query: &str, k: usize) -> Result<Vec<(usize, f32)>, AppError> { pub fn search(&self, query: &str, k: usize) -> Result<Vec<(usize, f32)>, AppError> {
let reader = self let searcher = self.reader.searcher();
.index
.reader()
.map_err(|e| AppError::Io(format!("tantivy reader: {e}")))?;
let searcher = reader.searcher();
let norm = normalize(query); let norm = normalize(query);
let sanitized = norm.replace(['(', ')', '"', ':'], " "); // Strip all tantivy grammar specials (-, +, ~, *, etc.) — medical text
// never intends them as operators; leaving them silently corrupts recall
// (e.g. "C50.-" triggers NOT, "hüft-tep" loses the second token).
let sanitized = norm.replace(
['(', ')', '"', ':', '-', '+', '[', ']', '{', '}', '~', '!', '^', '*', '\\', '?'],
" ",
);
// Use OR semantics (default) for recall-favoring behaviour; fall back to // Use OR semantics (default) for recall-favoring behaviour; fall back to
// the raw normalised query on parse error. // conjunction semantics on parse error — both attempts use the sanitized string.
let parsed = QueryParser::for_index(&self.index, vec![self.text_field]) let parsed = QueryParser::for_index(&self.index, vec![self.text_field])
.parse_query(&sanitized) .parse_query(&sanitized)
.or_else(|_| { .or_else(|_| {
let mut qp = QueryParser::for_index(&self.index, vec![self.text_field]); let mut qp = QueryParser::for_index(&self.index, vec![self.text_field]);
qp.set_conjunction_by_default(); qp.set_conjunction_by_default();
qp.parse_query(&norm) qp.parse_query(&sanitized)
}) })
.map_err(|e| AppError::Parse(format!("query: {e}")))?; .map_err(|e| AppError::Parse(format!("query: {e}")))?;
let top = searcher let top = searcher