settings/icu_tokenizer.yaml

   1 normalization:
   2     - ":: lower ()"
   3     - ":: Hans-Hant"
   4     - !include icu-rules/unicode-digits-to-decimal.yaml
   5     - "'№' > 'no'"
   6     - "'n°' > 'no'"
   7     - "'nº' > 'no'"
   8     - "ª > a"
   9     - "º > o"
  10     - "[[:Punctuation:][:Symbol:]\u02bc]  > ' '"
  11     - "ß > 'ss'" # German szet is unimbigiously equal to double ss
  12     - "[^[:Letter:] [:Number:] [:Space:]] >"
  13     - "[:Lm:] >"
  14     - ":: [[:Number:]] Latin ()"
  15     - ":: [[:Number:]] Ascii ();"
  16     - ":: [[:Number:]] NFD ();"
  17     - "[[:Nonspacing Mark:] [:Cf:]] >;"
  18     - "[:Space:]+ > ' '"
  19 transliteration:
  20     - ":: Latin ()"
  21     - !include icu-rules/extended-unicode-to-asccii.yaml
  22     - ":: Ascii ()"
  23     - ":: NFD ()"
  24     - ":: lower ()"
  25     - "[^a-z0-9[:Space:]] >"
  26     - ":: NFC ()"
  27 sanitizers:
  28     - step: split-name-list
  29     - step: strip-brace-terms
  30     - step: tag-analyzer-by-language
  31       filter-kind: [".*name.*"]
  32       whitelist: [bg,ca,cs,da,de,el,en,es,et,eu,fi,fr,gl,hu,it,ja,mg,ms,nl,no,pl,pt,ro,ru,sk,sl,sv,tr,uk,vi]
  33       use-defaults: all
  34       mode: append
  35 token-analysis:
  36     - analyzer: generic
  37     - id: bg
  38       analyzer: generic
  39       mode: variant-only
  40       variants:
  41           - !include icu-rules/variants-bg.yaml
  42     - id: ca
  43       analyzer: generic
  44       mode: variant-only
  45       variants:
  46           - !include icu-rules/variants-ca.yaml
  47     - id: cs
  48       analyzer: generic
  49       mode: variant-only
  50       variants:
  51           - !include icu-rules/variants-cs.yaml
  52     - id: da
  53       analyzer: generic
  54       mode: variant-only
  55       variants:
  56           - !include icu-rules/variants-da.yaml
  57     - id: de
  58       analyzer: generic
  59       mode: variant-only
  60       variants:
  61           - !include icu-rules/variants-de.yaml
  62       mutations:
  63           - pattern: ä
  64             replacements: ["ä", "ae"]
  65           - pattern: ö
  66             replacements: ["ö", "oe"]
  67           - pattern: ü
  68             replacements: ["ü", "ue"]
  69     - id: el
  70       analyzer: generic
  71       mode: variant-only
  72       variants:
  73           - !include icu-rules/variants-el.yaml
  74     - id: en
  75       analyzer: generic
  76       mode: variant-only
  77       variants:
  78           - !include icu-rules/variants-en.yaml
  79     - id: es
  80       analyzer: generic
  81       mode: variant-only
  82       variants:
  83           - !include icu-rules/variants-es.yaml
  84     - id: et
  85       analyzer: generic
  86       mode: variant-only
  87       variants:
  88           - !include icu-rules/variants-et.yaml
  89     - id: eu
  90       analyzer: generic
  91       mode: variant-only
  92       variants:
  93           - !include icu-rules/variants-eu.yaml
  94     - id: fi
  95       analyzer: generic
  96       mode: variant-only
  97       variants:
  98           - !include icu-rules/variants-fi.yaml
  99     - id: fr
 100       analyzer: generic
 101       mode: variant-only
 102       variants:
 103           - !include icu-rules/variants-fr.yaml
 104     - id: gl
 105       analyzer: generic
 106       mode: variant-only
 107       variants:
 108           - !include icu-rules/variants-gl.yaml
 109     - id: hu
 110       analyzer: generic
 111       mode: variant-only
 112       variants:
 113           - !include icu-rules/variants-hu.yaml
 114     - id: it
 115       analyzer: generic
 116       mode: variant-only
 117       variants:
 118           - !include icu-rules/variants-it.yaml
 119     - id: ja
 120       analyzer: generic
 121       mode: variant-only
 122       variants:
 123           - !include icu-rules/variants-ja.yaml
 124     - id: mg
 125       analyzer: generic
 126       mode: variant-only
 127       variants:
 128           - !include icu-rules/variants-mg.yaml
 129     - id: ms
 130       analyzer: generic
 131       mode: variant-only
 132       variants:
 133           - !include icu-rules/variants-ms.yaml
 134     - id: nl
 135       analyzer: generic
 136       mode: variant-only
 137       variants:
 138           - !include icu-rules/variants-nl.yaml
 139     - id: no
 140       analyzer: generic
 141       mode: variant-only
 142       variants:
 143           - !include icu-rules/variants-no.yaml
 144     - id: pl
 145       analyzer: generic
 146       mode: variant-only
 147       variants:
 148           - !include icu-rules/variants-pl.yaml
 149     - id: pt
 150       analyzer: generic
 151       mode: variant-only
 152       variants:
 153           - !include icu-rules/variants-pt.yaml
 154     - id: ro
 155       analyzer: generic
 156       mode: variant-only
 157       variants:
 158           - !include icu-rules/variants-ro.yaml
 159     - id: ru
 160       analyzer: generic
 161       mode: variant-only
 162       variants:
 163           - !include icu-rules/variants-ru.yaml
 164     - id: sk
 165       analyzer: generic
 166       mode: variant-only
 167       variants:
 168           - !include icu-rules/variants-sk.yaml
 169     - id: sl
 170       analyzer: generic
 171       mode: variant-only
 172       variants:
 173           - !include icu-rules/variants-sl.yaml
 174     - id: sv
 175       analyzer: generic
 176       mode: variant-only
 177       variants:
 178           - !include icu-rules/variants-sv.yaml
 179     - id: tr
 180       analyzer: generic
 181       mode: variant-only
 182       variants:
 183           - !include icu-rules/variants-tr.yaml
 184     - id: uk
 185       analyzer: generic
 186       mode: variant-only
 187       variants:
 188           - !include icu-rules/variants-uk.yaml
 189     - id: vi
 190       analyzer: generic
 191       mode: variant-only
 192       variants:
 193           - !include icu-rules/variants-vi.yaml