fix: store IndexReader once; sanitize all tantivy query specials
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+15
-8
@@ -7,6 +7,7 @@ use tantivy::{doc, Index, TantivyDocument};
|
|||||||
|
|
||||||
pub struct LexicalIndex {
|
pub struct LexicalIndex {
|
||||||
index: Index,
|
index: Index,
|
||||||
|
reader: tantivy::IndexReader,
|
||||||
text_field: tantivy::schema::Field,
|
text_field: tantivy::schema::Field,
|
||||||
idx_field: tantivy::schema::Field,
|
idx_field: tantivy::schema::Field,
|
||||||
}
|
}
|
||||||
@@ -31,8 +32,12 @@ impl LexicalIndex {
|
|||||||
}
|
}
|
||||||
w.commit()
|
w.commit()
|
||||||
.map_err(|e| AppError::Io(format!("tantivy commit: {e}")))?;
|
.map_err(|e| AppError::Io(format!("tantivy commit: {e}")))?;
|
||||||
|
let reader = index
|
||||||
|
.reader()
|
||||||
|
.map_err(|e| AppError::Io(format!("tantivy reader: {e}")))?;
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
index,
|
index,
|
||||||
|
reader,
|
||||||
text_field,
|
text_field,
|
||||||
idx_field,
|
idx_field,
|
||||||
})
|
})
|
||||||
@@ -40,21 +45,23 @@ impl LexicalIndex {
|
|||||||
|
|
||||||
/// Returns (entry_index, bm25_score) descending.
|
/// Returns (entry_index, bm25_score) descending.
|
||||||
pub fn search(&self, query: &str, k: usize) -> Result<Vec<(usize, f32)>, AppError> {
|
pub fn search(&self, query: &str, k: usize) -> Result<Vec<(usize, f32)>, AppError> {
|
||||||
let reader = self
|
let searcher = self.reader.searcher();
|
||||||
.index
|
|
||||||
.reader()
|
|
||||||
.map_err(|e| AppError::Io(format!("tantivy reader: {e}")))?;
|
|
||||||
let searcher = reader.searcher();
|
|
||||||
let norm = normalize(query);
|
let norm = normalize(query);
|
||||||
let sanitized = norm.replace(['(', ')', '"', ':'], " ");
|
// Strip all tantivy grammar specials (-, +, ~, *, etc.) — medical text
|
||||||
|
// never intends them as operators; leaving them silently corrupts recall
|
||||||
|
// (e.g. "C50.-" triggers NOT, "hüft-tep" loses the second token).
|
||||||
|
let sanitized = norm.replace(
|
||||||
|
['(', ')', '"', ':', '-', '+', '[', ']', '{', '}', '~', '!', '^', '*', '\\', '?'],
|
||||||
|
" ",
|
||||||
|
);
|
||||||
// Use OR semantics (default) for recall-favoring behaviour; fall back to
|
// Use OR semantics (default) for recall-favoring behaviour; fall back to
|
||||||
// the raw normalised query on parse error.
|
// conjunction semantics on parse error — both attempts use the sanitized string.
|
||||||
let parsed = QueryParser::for_index(&self.index, vec![self.text_field])
|
let parsed = QueryParser::for_index(&self.index, vec![self.text_field])
|
||||||
.parse_query(&sanitized)
|
.parse_query(&sanitized)
|
||||||
.or_else(|_| {
|
.or_else(|_| {
|
||||||
let mut qp = QueryParser::for_index(&self.index, vec![self.text_field]);
|
let mut qp = QueryParser::for_index(&self.index, vec![self.text_field]);
|
||||||
qp.set_conjunction_by_default();
|
qp.set_conjunction_by_default();
|
||||||
qp.parse_query(&norm)
|
qp.parse_query(&sanitized)
|
||||||
})
|
})
|
||||||
.map_err(|e| AppError::Parse(format!("query: {e}")))?;
|
.map_err(|e| AppError::Parse(format!("query: {e}")))?;
|
||||||
let top = searcher
|
let top = searcher
|
||||||
|
|||||||
Reference in New Issue
Block a user