diff options
Diffstat (limited to 'vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer')
2 files changed, 47 insertions, 0 deletions
diff --git a/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/TokenizerInterface.php b/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/TokenizerInterface.php new file mode 100644 index 000000000..f06074628 --- /dev/null +++ b/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/TokenizerInterface.php @@ -0,0 +1,18 @@ +<?php + +declare(strict_types = 1); + +namespace LanguageDetection\Tokenizer; + +/** + * Interface TokenizerInterface + * + * @copyright Patrick Schur + * @license https://opensource.org/licenses/mit-license.html MIT + * @author Patrick Schur <patrick_schur@outlook.de> + * @package LanguageDetection + */ +interface TokenizerInterface +{ + public function tokenize(string $str): array; +}
\ No newline at end of file diff --git a/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php b/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php new file mode 100644 index 000000000..68bb6013a --- /dev/null +++ b/vendor/patrickschur/language-detection/src/LanguageDetection/Tokenizer/WhitespaceTokenizer.php @@ -0,0 +1,29 @@ +<?php + +declare(strict_types = 1); + +namespace LanguageDetection\Tokenizer; + +/** + * Class WhitespaceTokenizer + * + * @copyright Patrick Schur + * @license https://opensource.org/licenses/mit-license.html MIT + * @author Patrick Schur <patrick_schur@outlook.de> + * @package LanguageDetection + */ +class WhitespaceTokenizer implements TokenizerInterface +{ + /** + * @param string $str + * @return array + */ + public function tokenize(string $str): array + { + return \array_map(function ($word) { + return "_{$word}_"; + }, + \preg_split('/[^\pL]+(?<![\x27\x60\x{2019}])/u', $str, -1, PREG_SPLIT_NO_EMPTY) + ); + } +}
\ No newline at end of file |