From e67ae701ac76129d24578cc70b8e25987a162a81 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 28 Feb 2025 17:19:37 +0100 Subject: [PATCH] show token begin and end in debug output --- src/nominatim_api/search/icu_tokenizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nominatim_api/search/icu_tokenizer.py b/src/nominatim_api/search/icu_tokenizer.py index 01513103..09827826 100644 --- a/src/nominatim_api/search/icu_tokenizer.py +++ b/src/nominatim_api/search/icu_tokenizer.py @@ -298,12 +298,12 @@ class ICUQueryAnalyzer(AbstractQueryAnalyzer): def _dump_word_tokens(query: qmod.QueryStruct) -> Iterator[List[Any]]: - yield ['type', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info'] - for node in query.nodes: + yield ['type', 'from', 'to', 'token', 'word_token', 'lookup_word', 'penalty', 'count', 'info'] + for i, node in enumerate(query.nodes): for tlist in node.starting: for token in tlist.tokens: t = cast(ICUToken, token) - yield [tlist.ttype, t.token, t.word_token or '', + yield [tlist.ttype, str(i), str(tlist.end), t.token, t.word_token or '', t.lookup_word or '', t.penalty, t.count, t.info] -- 2.39.5