settings/icu_tokenizer.yaml

   1 query-preprocessing:
   2     - step: split_japanese_phrases
   3     - step: regex_replace
   4       replacements:
   5         - pattern: \b(?:25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(?:\.(?:25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}\b # Filter for IPv4 addresses
   6           replace: ''
   7         - pattern: \b(?:(?:[A-Fa-f0-9]{1,4}:){1,7}|:)(?:[A-Fa-f0-9]{1,4})?\b # Filter for IPv6 addresses
   8           replace: ''
   9         - pattern: https?://[^\s]* # Filter URLs starting with http or https
  10           replace: ''
  11     - step: normalize
  12 normalization:
  13     - ":: lower ()"
  14     - ":: Hans-Hant"
  15     - !include icu-rules/unicode-digits-to-decimal.yaml
  16     - "'№' > 'no'"
  17     - "'n°' > 'no'"
  18     - "'nº' > 'no'"
  19     - "ª > a"
  20     - "º > o"
  21     - "[[:Punctuation:][:Symbol:][\u02bc] - [-:]]+  > '-'"
  22     - "ß > 'ss'" # German szet is unambiguously equal to double ss
  23     - "[^[:alnum:] [:Canonical_Combining_Class=Virama:] [:Space:] [-:]] >"
  24     - "[:Lm:] >"
  25     - ":: [[:Number:]] Latin ()"
  26     - ":: [[:Number:]] Ascii ();"
  27     - ":: [[:Number:]] NFD ();"
  28     - "[[:Nonspacing Mark:] [:Cf:]] >;"
  29     - "[-:]?[:Space:]+[-:]? > ' '"
  30 transliteration:
  31     - "[-:]  > ' '"
  32     - ":: Latin ()"
  33     - !include icu-rules/extended-unicode-to-asccii.yaml
  34     - ":: Ascii ()"
  35     - ":: NFD ()"
  36     - ":: lower ()"
  37     - "[^a-z0-9[:Space:]] >"
  38     - ":: NFC ()"
  39     - "[:Space:]+ > ' '"
  40 sanitizers:
  41     - step: clean-housenumbers
  42       filter-kind:
  43         - housenumber
  44         - conscriptionnumber
  45         - streetnumber
  46       convert-to-name:
  47         - (\A|.*,)[^\d,]{3,}(,.*|\Z)
  48     - step: clean-postcodes
  49       convert-to-address: yes
  50       default-pattern: "[A-Z0-9- ]{3,12}"
  51     - step: clean-tiger-tags
  52     - step: split-name-list
  53       delimiters: ;
  54     - step: strip-brace-terms
  55     - step: tag-analyzer-by-language
  56       filter-kind: [".*name.*"]
  57       whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,"no",pl,pt,ro,ru,sk,sl,sv,tr,uk,vi]
  58       use-defaults: all
  59       mode: append
  60     - step: tag-japanese
  61 token-analysis:
  62     - analyzer: generic
  63     - id: "@housenumber"
  64       analyzer: housenumbers
  65     - id: "@postcode"
  66       analyzer: postcodes
  67     - id: bg
  68       analyzer: generic
  69       mode: variant-only
  70       variants:
  71           - !include icu-rules/variants-bg.yaml
  72     - id: ca
  73       analyzer: generic
  74       mode: variant-only
  75       variants:
  76           - !include icu-rules/variants-ca.yaml
  77     - id: cs
  78       analyzer: generic
  79       mode: variant-only
  80       variants:
  81           - !include icu-rules/variants-cs.yaml
  82     - id: da
  83       analyzer: generic
  84       mode: variant-only
  85       variants:
  86           - !include icu-rules/variants-da.yaml
  87     - id: de
  88       analyzer: generic
  89       mode: variant-only
  90       variants:
  91           - !include icu-rules/variants-de.yaml
  92       mutations:
  93           - pattern: ä
  94             replacements: ["ä", "ae"]
  95           - pattern: ö
  96             replacements: ["ö", "oe"]
  97           - pattern: ü
  98             replacements: ["ü", "ue"]
  99     - id: el
 100       analyzer: generic
 101       mode: variant-only
 102       variants:
 103           - !include icu-rules/variants-el.yaml
 104     - id: en
 105       analyzer: generic
 106       mode: variant-only
 107       variants:
 108           - !include icu-rules/variants-en.yaml
 109     - id: es
 110       analyzer: generic
 111       mode: variant-only
 112       variants:
 113           - !include icu-rules/variants-es.yaml
 114     - id: et
 115       analyzer: generic
 116       mode: variant-only
 117       variants:
 118           - !include icu-rules/variants-et.yaml
 119     - id: eu
 120       analyzer: generic
 121       mode: variant-only
 122       variants:
 123           - !include icu-rules/variants-eu.yaml
 124     - id: fi
 125       analyzer: generic
 126       mode: variant-only
 127       variants:
 128           - !include icu-rules/variants-fi.yaml
 129     - id: fr
 130       analyzer: generic
 131       mode: variant-only
 132       variants:
 133           - !include icu-rules/variants-fr.yaml
 134     - id: gl
 135       analyzer: generic
 136       mode: variant-only
 137       variants:
 138           - !include icu-rules/variants-gl.yaml
 139     - id: hu
 140       analyzer: generic
 141       mode: variant-only
 142       variants:
 143           - !include icu-rules/variants-hu.yaml
 144     - id: it
 145       analyzer: generic
 146       mode: variant-only
 147       variants:
 148           - !include icu-rules/variants-it.yaml
 149     - id: ja
 150       analyzer: generic
 151       mode: variant-only
 152       variants:
 153           - !include icu-rules/variants-ja.yaml
 154     - id: mg
 155       analyzer: generic
 156       mode: variant-only
 157       variants:
 158           - !include icu-rules/variants-mg.yaml
 159     - id: ms
 160       analyzer: generic
 161       mode: variant-only
 162       variants:
 163           - !include icu-rules/variants-ms.yaml
 164     - id: nl
 165       analyzer: generic
 166       mode: variant-only
 167       variants:
 168           - !include icu-rules/variants-nl.yaml
 169     - id: "no"
 170       analyzer: generic
 171       mode: variant-only
 172       variants:
 173           - !include icu-rules/variants-no.yaml
 174     - id: pl
 175       analyzer: generic
 176       mode: variant-only
 177       variants:
 178           - !include icu-rules/variants-pl.yaml
 179     - id: pt
 180       analyzer: generic
 181       mode: variant-only
 182       variants:
 183           - !include icu-rules/variants-pt.yaml
 184     - id: ro
 185       analyzer: generic
 186       mode: variant-only
 187       variants:
 188           - !include icu-rules/variants-ro.yaml
 189     - id: ru
 190       analyzer: generic
 191       mode: variant-only
 192       variants:
 193           - !include icu-rules/variants-ru.yaml
 194     - id: sk
 195       analyzer: generic
 196       mode: variant-only
 197       variants:
 198           - !include icu-rules/variants-sk.yaml
 199     - id: sl
 200       analyzer: generic
 201       mode: variant-only
 202       variants:
 203           - !include icu-rules/variants-sl.yaml
 204     - id: sv
 205       analyzer: generic
 206       mode: variant-only
 207       variants:
 208           - !include icu-rules/variants-sv.yaml
 209     - id: tr
 210       analyzer: generic
 211       mode: variant-only
 212       variants:
 213           - !include icu-rules/variants-tr.yaml
 214     - id: uk
 215       analyzer: generic
 216       mode: variant-only
 217       variants:
 218           - !include icu-rules/variants-uk.yaml
 219     - id: vi
 220       analyzer: generic
 221       mode: variant-only
 222       variants:
 223           - !include icu-rules/variants-vi.yaml