fix: store IndexReader once; sanitize all tantivy query specials
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+15
-8
@@ -7,6 +7,7 @@ use tantivy::{doc, Index, TantivyDocument};
|
||||
|
||||
pub struct LexicalIndex {
|
||||
index: Index,
|
||||
reader: tantivy::IndexReader,
|
||||
text_field: tantivy::schema::Field,
|
||||
idx_field: tantivy::schema::Field,
|
||||
}
|
||||
@@ -31,8 +32,12 @@ impl LexicalIndex {
|
||||
}
|
||||
w.commit()
|
||||
.map_err(|e| AppError::Io(format!("tantivy commit: {e}")))?;
|
||||
let reader = index
|
||||
.reader()
|
||||
.map_err(|e| AppError::Io(format!("tantivy reader: {e}")))?;
|
||||
Ok(Self {
|
||||
index,
|
||||
reader,
|
||||
text_field,
|
||||
idx_field,
|
||||
})
|
||||
@@ -40,21 +45,23 @@ impl LexicalIndex {
|
||||
|
||||
/// Returns (entry_index, bm25_score) descending.
|
||||
pub fn search(&self, query: &str, k: usize) -> Result<Vec<(usize, f32)>, AppError> {
|
||||
let reader = self
|
||||
.index
|
||||
.reader()
|
||||
.map_err(|e| AppError::Io(format!("tantivy reader: {e}")))?;
|
||||
let searcher = reader.searcher();
|
||||
let searcher = self.reader.searcher();
|
||||
let norm = normalize(query);
|
||||
let sanitized = norm.replace(['(', ')', '"', ':'], " ");
|
||||
// Strip all tantivy grammar specials (-, +, ~, *, etc.) — medical text
|
||||
// never intends them as operators; leaving them silently corrupts recall
|
||||
// (e.g. "C50.-" triggers NOT, "hüft-tep" loses the second token).
|
||||
let sanitized = norm.replace(
|
||||
['(', ')', '"', ':', '-', '+', '[', ']', '{', '}', '~', '!', '^', '*', '\\', '?'],
|
||||
" ",
|
||||
);
|
||||
// Use OR semantics (default) for recall-favoring behaviour; fall back to
|
||||
// the raw normalised query on parse error.
|
||||
// conjunction semantics on parse error — both attempts use the sanitized string.
|
||||
let parsed = QueryParser::for_index(&self.index, vec![self.text_field])
|
||||
.parse_query(&sanitized)
|
||||
.or_else(|_| {
|
||||
let mut qp = QueryParser::for_index(&self.index, vec![self.text_field]);
|
||||
qp.set_conjunction_by_default();
|
||||
qp.parse_query(&norm)
|
||||
qp.parse_query(&sanitized)
|
||||
})
|
||||
.map_err(|e| AppError::Parse(format!("query: {e}")))?;
|
||||
let top = searcher
|
||||
|
||||
Reference in New Issue
Block a user