diff options
author | Mario <mario@mariovavti.com> | 2023-10-07 16:00:34 +0000 |
---|---|---|
committer | Mario <mario@mariovavti.com> | 2023-10-07 16:00:34 +0000 |
commit | 2f2e353ecef52639a78cac3bc407ccfe64197ac9 (patch) | |
tree | 557793121ec08dce6e97c064a100994e8d3990c5 /vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php | |
parent | 0092b7c0a4d6cf49c092e2232af63f87be63142b (diff) | |
download | volse-hubzilla-2f2e353ecef52639a78cac3bc407ccfe64197ac9.tar.gz volse-hubzilla-2f2e353ecef52639a78cac3bc407ccfe64197ac9.tar.bz2 volse-hubzilla-2f2e353ecef52639a78cac3bc407ccfe64197ac9.zip |
use new lang detect library which supports much more languages
Diffstat (limited to 'vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php')
-rw-r--r-- | vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php | 29 |
1 files changed, 29 insertions, 0 deletions
diff --git a/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php b/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php new file mode 100644 index 000000000..68bb6013a --- /dev/null +++ b/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php @@ -0,0 +1,29 @@ +<?php + +declare(strict_types = 1); + +namespace LanguageDetection\Tokenizer; + +/** + * Class WhitespaceTokenizer + * + * @copyright Patrick Schur + * @license https://opensource.org/licenses/mit-license.html MIT + * @author Patrick Schur <patrick_schur@outlook.de> + * @package LanguageDetection + */ +class WhitespaceTokenizer implements TokenizerInterface +{ + /** + * @param string $str + * @return array + */ + public function tokenize(string $str): array + { + return \array_map(function ($word) { + return "_{$word}_"; + }, + \preg_split('/[^\pL]+(?<![\x27\x60\x{2019}])/u', $str, -1, PREG_SPLIT_NO_EMPTY) + ); + } +}
\ No newline at end of file |