diff options
Diffstat (limited to 'library/langdet/Text')
| -rw-r--r-- | library/langdet/Text/LanguageDetect.php | 1708 | ||||
| -rw-r--r-- | library/langdet/Text/LanguageDetect/Exception.php | 57 | ||||
| -rw-r--r-- | library/langdet/Text/LanguageDetect/ISO639.php | 341 | ||||
| -rw-r--r-- | library/langdet/Text/LanguageDetect/Parser.php | 349 |
4 files changed, 2455 insertions, 0 deletions
diff --git a/library/langdet/Text/LanguageDetect.php b/library/langdet/Text/LanguageDetect.php new file mode 100644 index 000000000..7cebbe607 --- /dev/null +++ b/library/langdet/Text/LanguageDetect.php @@ -0,0 +1,1708 @@ +<?php + +/** + * Detects the language of a given piece of text. + * + * Attempts to detect the language of a sample of text by correlating ranked + * 3-gram frequencies to a table of 3-gram frequencies of known languages. + * + * Implements a version of a technique originally proposed by Cavnar & Trenkle + * (1994): "N-Gram-Based Text Categorization" + * + * PHP version 5 + * + * @category Text + * @package Text_LanguageDetect + * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com> + * @copyright 2005-2006 Nicholas Pisarro + * @license http://www.debian.org/misc/bsd.license BSD + * @version SVN: $Id: LanguageDetect.php 322353 2012-01-16 08:41:43Z cweiske $ + * @link http://pear.php.net/package/Text_LanguageDetect/ + * @link http://langdetect.blogspot.com/ + */ + +require_once 'Text/LanguageDetect/Exception.php'; +require_once 'Text/LanguageDetect/Parser.php'; +require_once 'Text/LanguageDetect/ISO639.php'; + +/** + * Language detection class + * + * Requires the langauge model database (lang.dat) that should have + * accompanied this class definition in order to be instantiated. + * + * Example usage: + * + * <code> + * require_once 'Text/LanguageDetect.php'; + * + * $l = new Text_LanguageDetect; + * + * $stdin = fopen('php://stdin', 'r'); + * + * echo "Supported languages:\n"; + * + * try { + * $langs = $l->getLanguages(); + * } catch (Text_LanguageDetect_Exception $e) { + * die($e->getMessage()); + * } + * + * sort($langs); + * echo join(', ', $langs); + * + * while ($line = fgets($stdin)) { + * print_r($l->detect($line, 4)); + * } + * </code> + * + * @category Text + * @package Text_LanguageDetect + * @author Nicholas Pisarro <infinityminusnine+pear@gmail.com> + * @copyright 2005 Nicholas Pisarro + * @license http://www.debian.org/misc/bsd.license BSD + * @version Release: @package_version@ + * @link http://pear.php.net/package/Text_LanguageDetect/ + * @todo allow users to generate their own language models + */ +class Text_LanguageDetect +{ + /** + * The filename that stores the trigram data for the detector + * + * If this value starts with a slash (/) or a dot (.) the value of + * $this->_data_dir will be ignored + * + * @var string + * @access private + */ + var $_db_filename = 'lang.dat'; + + /** + * The filename that stores the unicode block definitions + * + * If this value starts with a slash (/) or a dot (.) the value of + * $this->_data_dir will be ignored + * + * @var string + * @access private + */ + var $_unicode_db_filename = 'unicode_blocks.dat'; + + /** + * The data directory + * + * Should be set by PEAR installer + * + * @var string + * @access private + */ + var $_data_dir = '@data_dir@'; + + /** + * The trigram data for comparison + * + * Will be loaded on start from $this->_db_filename + * + * @var array + * @access private + */ + var $_lang_db = array(); + + /** + * stores the map of the trigram data to unicode characters + * + * @access private + * @var array + */ + var $_unicode_map; + + /** + * The size of the trigram data arrays + * + * @var int + * @access private + */ + var $_threshold = 300; + + /** + * the maximum possible score. + * + * needed for score normalization. Different depending on the + * perl compatibility setting + * + * @access private + * @var int + * @see setPerlCompatible() + */ + var $_max_score = 0; + + /** + * Whether or not to simulate perl's Language::Guess exactly + * + * @access private + * @var bool + * @see setPerlCompatible() + */ + var $_perl_compatible = false; + + /** + * Whether to use the unicode block detection to speed up processing + * + * @access private + * @var bool + */ + var $_use_unicode_narrowing = true; + + /** + * stores the result of the clustering operation + * + * @access private + * @var array + * @see clusterLanguages() + */ + var $_clusters; + + /** + * Which type of "language names" are accepted and returned: + * + * 0 - language name ("english") + * 2 - 2-letter ISO 639-1 code ("en") + * 3 - 3-letter ISO 639-2 code ("eng") + */ + var $_name_mode = 0; + + /** + * Constructor + * + * Will attempt to load the language database. If it fails, you will get + * an exception. + */ + function __construct() + { + $data = $this->_readdb($this->_db_filename); + $this->_checkTrigram($data['trigram']); + $this->_lang_db = $data['trigram']; + + if (isset($data['trigram-unicodemap'])) { + $this->_unicode_map = $data['trigram-unicodemap']; + } + + // Not yet implemented: + if (isset($data['trigram-clusters'])) { + $this->_clusters = $data['trigram-clusters']; + } + } + + /** + * Returns the path to the location of the database + * + * @param string $fname File name to load + * + * @return string expected path to the language model database + * @access private + */ + function _get_data_loc($fname) + { + if ($fname{0} == '/' || $fname{0} == '.') { + // if filename starts with a slash, assume it's an absolute pathname + // and skip whatever is in $this->_data_dir + return $fname; + + } elseif ($this->_data_dir != '@' . 'data_dir' . '@') { + // if the data dir was set by the PEAR installer, use that + return $this->_data_dir . '/Text_LanguageDetect/' . $fname; + + } else { + // assume this was just unpacked somewhere + // try the local working directory if otherwise + return __DIR__ . '/../data/' . $fname; + } + } + + /** + * Loads the language trigram database from filename + * + * Trigram datbase should be a serialize()'d array + * + * @param string $fname the filename where the data is stored + * + * @return array the language model data + * @throws Text_LanguageDetect_Exception + * @access private + */ + function _readdb($fname) + { + // finds the correct data dir + $fname = $this->_get_data_loc($fname); + + // input check + if (!file_exists($fname)) { + throw new Text_LanguageDetect_Exception( + 'Language database does not exist: ' . $fname, + Text_LanguageDetect_Exception::DB_NOT_FOUND + ); + } elseif (!is_readable($fname)) { + throw new Text_LanguageDetect_Exception( + 'Language database is not readable: ' . $fname, + Text_LanguageDetect_Exception::DB_NOT_READABLE + ); + } + + return unserialize(file_get_contents($fname)); + } + + + /** + * Checks if this object is ready to detect languages + * + * @param array $trigram Trigram data from database + * + * @return void + * @access private + */ + function _checkTrigram($trigram) + { + if (!is_array($trigram)) { + if (ini_get('magic_quotes_runtime')) { + throw new Text_LanguageDetect_Exception( + 'Error loading database. Try turning magic_quotes_runtime off.', + Text_LanguageDetect_Exception::MAGIC_QUOTES + ); + } + throw new Text_LanguageDetect_Exception( + 'Language database is not an array.', + Text_LanguageDetect_Exception::DB_NOT_ARRAY + ); + } elseif (empty($trigram)) { + throw new Text_LanguageDetect_Exception( + 'Language database has no elements.', + Text_LanguageDetect_Exception::DB_EMPTY + ); + } + } + + /** + * Omits languages + * + * Pass this function the name of or an array of names of + * languages that you don't want considered + * + * If you're only expecting a limited set of languages, this can greatly + * speed up processing + * + * @param mixed $omit_list language name or array of names to omit + * @param bool $include_only if true will include (rather than + * exclude) only those in the list + * + * @return int number of languages successfully deleted + * @throws Text_LanguageDetect_Exception + */ + public function omitLanguages($omit_list, $include_only = false) + { + $deleted = 0; + + $omit_list = $this->_convertFromNameMode($omit_list); + + if (!$include_only) { + // deleting the given languages + if (!is_array($omit_list)) { + $omit_list = strtolower($omit_list); // case desensitize + if (isset($this->_lang_db[$omit_list])) { + unset($this->_lang_db[$omit_list]); + $deleted++; + } + } else { + foreach ($omit_list as $omit_lang) { + if (isset($this->_lang_db[$omit_lang])) { + unset($this->_lang_db[$omit_lang]); + $deleted++; + } + } + } + + } else { + // deleting all except the given languages + if (!is_array($omit_list)) { + $omit_list = array($omit_list); + } + + // case desensitize + foreach ($omit_list as $key => $omit_lang) { + $omit_list[$key] = strtolower($omit_lang); + } + + foreach (array_keys($this->_lang_db) as $lang) { + if (!in_array($lang, $omit_list)) { + unset($this->_lang_db[$lang]); + $deleted++; + } + } + } + + // reset the cluster cache if the number of languages changes + // this will then have to be recalculated + if (isset($this->_clusters) && $deleted > 0) { + $this->_clusters = null; + } + + return $deleted; + } + + + /** + * Returns the number of languages that this object can detect + * + * @access public + * @return int the number of languages + * @throws Text_LanguageDetect_Exception + */ + function getLanguageCount() + { + return count($this->_lang_db); + } + + /** + * Checks if the language with the given name exists in the database + * + * @param mixed $lang Language name or array of language names + * + * @return bool true if language model exists + */ + public function languageExists($lang) + { + $lang = $this->_convertFromNameMode($lang); + + if (is_string($lang)) { + return isset($this->_lang_db[strtolower($lang)]); + + } elseif (is_array($lang)) { + foreach ($lang as $test_lang) { + if (!isset($this->_lang_db[strtolower($test_lang)])) { + return false; + } + } + return true; + + } else { + throw new Text_LanguageDetect_Exception( + 'Unsupported parameter type passed to languageExists()', + Text_LanguageDetect_Exception::PARAM_TYPE + ); + } + } + + /** + * Returns the list of detectable languages + * + * @access public + * @return array the names of the languages known to this object<<<<<<< + * @throws Text_LanguageDetect_Exception + */ + function getLanguages() + { + return $this->_convertToNameMode( + array_keys($this->_lang_db) + ); + } + + /** + * Make this object behave like Language::Guess + * + * @param bool $setting false to turn off perl compatibility + * + * @return void + */ + public function setPerlCompatible($setting = true) + { + if (is_bool($setting)) { // input check + $this->_perl_compatible = $setting; + + if ($setting == true) { + $this->_max_score = $this->_threshold; + } else { + $this->_max_score = 0; + } + } + + } + + /** + * Sets the way how language names are accepted and returned. + * + * @param integer $name_mode One of the following modes: + * 0 - language name ("english") + * 2 - 2-letter ISO 639-1 code ("en") + * 3 - 3-letter ISO 639-2 code ("eng") + * + * @return void + */ + function setNameMode($name_mode) + { + $this->_name_mode = $name_mode; + } + + /** + * Whether to use unicode block ranges in detection + * + * Should speed up most detections if turned on (detault is on). In some + * circumstances it may be slower, such as for large text samples (> 10K) + * in languages that use latin scripts. In other cases it should speed up + * detection noticeably. + * + * @param bool $setting false to turn off + * + * @return void + */ + public function useUnicodeBlocks($setting = true) + { + if (is_bool($setting)) { + $this->_use_unicode_narrowing = $setting; + } + } + + /** + * Converts a piece of text into trigrams + * + * @param string $text text to convert + * + * @return array array of trigram frequencies + * @access private + * @deprecated Superceded by the Text_LanguageDetect_Parser class + */ + function _trigram($text) + { + $s = new Text_LanguageDetect_Parser($text); + $s->prepareTrigram(); + $s->prepareUnicode(false); + $s->setPadStart(!$this->_perl_compatible); + $s->analyze(); + return $s->getTrigramFreqs(); + } + + /** + * Converts a set of trigrams from frequencies to ranks + * + * Thresholds (cuts off) the list at $this->_threshold + * + * @param array $arr array of trigram + * + * @return array ranks of trigrams + * @access protected + */ + function _arr_rank($arr) + { + + // sorts alphabetically first as a standard way of breaking rank ties + $this->_bub_sort($arr); + + // below might also work, but seemed to introduce errors in testing + //ksort($arr); + //asort($arr); + + $rank = array(); + + $i = 0; + foreach ($arr as $key => $value) { + $rank[$key] = $i++; + + // cut off at a standard threshold + if ($i >= $this->_threshold) { + break; + } + } + + return $rank; + } + + /** + * Sorts an array by value breaking ties alphabetically + * + * @param array &$arr the array to sort + * + * @return void + * @access private + */ + function _bub_sort(&$arr) + { + // should do the same as this perl statement: + // sort { $trigrams{$b} == $trigrams{$a} + // ? $a cmp $b : $trigrams{$b} <=> $trigrams{$a} } + + // needs to sort by both key and value at once + // using the key to break ties for the value + + // converts array into an array of arrays of each key and value + // may be a better way of doing this + $combined = array(); + + foreach ($arr as $key => $value) { + $combined[] = array($key, $value); + } + + usort($combined, array($this, '_sort_func')); + + $replacement = array(); + foreach ($combined as $key => $value) { + list($new_key, $new_value) = $value; + $replacement[$new_key] = $new_value; + } + + $arr = $replacement; + } + + /** + * Sort function used by bubble sort + * + * Callback function for usort(). + * + * @param array $a first param passed by usort() + * @param array $b second param passed by usort() + * + * @return int 1 if $a is greater, -1 if not + * @see _bub_sort() + * @access private + */ + function _sort_func($a, $b) + { + // each is actually a key/value pair, so that it can compare using both + list($a_key, $a_value) = $a; + list($b_key, $b_value) = $b; + + if ($a_value == $b_value) { + // if the values are the same, break ties using the key + return strcmp($a_key, $b_key); + + } else { + // if not, just sort normally + if ($a_value > $b_value) { + return -1; + } else { + return 1; + } + } + + // 0 should not be possible because keys must be unique + } + + /** + * Calculates a linear rank-order distance statistic between two sets of + * ranked trigrams + * + * Sums the differences in rank for each trigram. If the trigram does not + * appear in both, consider it a difference of $this->_threshold. + * + * This distance measure was proposed by Cavnar & Trenkle (1994). Despite + * its simplicity it has been shown to be highly accurate for language + * identification tasks. + * + * @param array $arr1 the reference set of trigram ranks + * @param array $arr2 the target set of trigram ranks + * + * @return int the sum of the differences between the ranks of + * the two trigram sets + * @access private + */ + function _distance($arr1, $arr2) + { + $sumdist = 0; + + foreach ($arr2 as $key => $value) { + if (isset($arr1[$key])) { + $distance = abs($value - $arr1[$key]); + } else { + // $this->_threshold sets the maximum possible distance value + // for any one pair of trigrams + $distance = $this->_threshold; + } + $sumdist += $distance; + } + + return $sumdist; + + // todo: there are other distance statistics to try, e.g. relative + // entropy, but they're probably more costly to compute + } + + /** + * Normalizes the score returned by _distance() + * + * Different if perl compatible or not + * + * @param int $score the score from _distance() + * @param int $base_count the number of trigrams being considered + * + * @return float the normalized score + * @see _distance() + * @access private + */ + function _normalize_score($score, $base_count = null) + { + if ($base_count === null) { + $base_count = $this->_threshold; + } + + if (!$this->_perl_compatible) { + return 1 - ($score / $base_count / $this->_threshold); + } else { + return floor($score / $base_count); + } + } + + + /** + * Detects the closeness of a sample of text to the known languages + * + * Calculates the statistical difference between the text and + * the trigrams for each language, normalizes the score then + * returns results for all languages in sorted order + * + * If perl compatible, the score is 300-0, 0 being most similar. + * Otherwise, it's 0-1 with 1 being most similar. + * + * The $sample text should be at least a few sentences in length; + * should be ascii-7 or utf8 encoded, if another and the mbstring extension + * is present it will try to detect and convert. However, experience has + * shown that mb_detect_encoding() *does not work very well* with at least + * some types of encoding. + * + * @param string $sample a sample of text to compare. + * @param int $limit if specified, return an array of the most likely + * $limit languages and their scores. + * + * @return mixed sorted array of language scores, blank array if no + * useable text was found + * @see _distance() + * @throws Text_LanguageDetect_Exception + */ + public function detect($sample, $limit = 0) + { + // input check + if (!Text_LanguageDetect_Parser::validateString($sample)) { + return array(); + } + + // check char encoding + // (only if mbstring extension is compiled and PHP > 4.0.6) + if (function_exists('mb_detect_encoding') + && function_exists('mb_convert_encoding') + ) { + // mb_detect_encoding isn't very reliable, to say the least + // detection should still work with a sufficient sample + // of ascii characters + $encoding = mb_detect_encoding($sample); + + // mb_detect_encoding() will return FALSE if detection fails + // don't attempt conversion if that's the case + if ($encoding != 'ASCII' && $encoding != 'UTF-8' + && $encoding !== false + ) { + // verify the encoding exists in mb_list_encodings + if (in_array($encoding, mb_list_encodings())) { + $sample = mb_convert_encoding($sample, 'UTF-8', $encoding); + } + } + } + + $sample_obj = new Text_LanguageDetect_Parser($sample); + $sample_obj->prepareTrigram(); + if ($this->_use_unicode_narrowing) { + $sample_obj->prepareUnicode(); + } + $sample_obj->setPadStart(!$this->_perl_compatible); + $sample_obj->analyze(); + + $trigram_freqs =& $sample_obj->getTrigramRanks(); + $trigram_count = count($trigram_freqs); + + if ($trigram_count == 0) { + return array(); + } + + $scores = array(); + + // use unicode block detection to narrow down the possibilities + if ($this->_use_unicode_narrowing) { + $blocks =& $sample_obj->getUnicodeBlocks(); + + if (is_array($blocks)) { + $present_blocks = array_keys($blocks); + } else { + throw new Text_LanguageDetect_Exception( + 'Error during block detection', + Text_LanguageDetect_Exception::BLOCK_DETECTION + ); + } + + $possible_langs = array(); + + foreach ($present_blocks as $blockname) { + if (isset($this->_unicode_map[$blockname])) { + + $possible_langs = array_merge( + $possible_langs, + array_keys($this->_unicode_map[$blockname]) + ); + + // todo: faster way to do this? + } + } + + // could also try an intersect operation rather than a union + // in other words, choose languages whose trigrams contain + // ALL of the unicode blocks found in this sample + // would improve speed but would be completely thrown off by an + // unexpected character, like an umlaut appearing in english text + + $possible_langs = array_intersect( + array_keys($this->_lang_db), + array_unique($possible_langs) + ); + + // needs to intersect it with the keys of _lang_db in case + // languages have been omitted + + } else { + // or just try 'em all + $possible_langs = array_keys($this->_lang_db); + } + + + foreach ($possible_langs as $lang) { + $scores[$lang] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang], $trigram_freqs), + $trigram_count + ); + } + + unset($sample_obj); + + if ($this->_perl_compatible) { + asort($scores); + } else { + arsort($scores); + } + + // todo: drop languages with a score of $this->_max_score? + + // limit the number of returned scores + if ($limit && is_numeric($limit)) { + $limited_scores = array(); + + $i = 0; + foreach ($scores as $key => $value) { + if ($i++ >= $limit) { + break; + } + + $limited_scores[$key] = $value; + } + + return $this->_convertToNameMode($limited_scores, true); + } else { + return $this->_convertToNameMode($scores, true); + } + } + + /** + * Returns only the most similar language to the text sample + * + * Calls $this->detect() and returns only the top result + * + * @param string $sample text to detect the language of + * + * @return string the name of the most likely language + * or null if no language is similar + * @see detect() + * @throws Text_LanguageDetect_Exception + */ + public function detectSimple($sample) + { + $scores = $this->detect($sample, 1); + + // if top language has the maximum possible score, + // then the top score will have been picked at random + if (!is_array($scores) || empty($scores) + || current($scores) == $this->_max_score + ) { + return null; + } else { + return key($scores); + } + } + + /** + * Returns an array containing the most similar language and a confidence + * rating + * + * Confidence is a simple measure calculated from the similarity score + * minus the similarity score from the next most similar language + * divided by the highest possible score. Languages that have closely + * related cousins (e.g. Norwegian and Danish) should generally have lower + * confidence scores. + * + * The similarity score answers the question "How likely is the text the + * returned language regardless of the other languages considered?" The + * confidence score is one way of answering the question "how likely is the + * text the detected language relative to the rest of the language model + * set?" + * + * To see how similar languages are a priori, see languageSimilarity() + * + * @param string $sample text for which language will be detected + * + * @return array most similar language, score and confidence rating + * or null if no language is similar + * @see detect() + * @throws Text_LanguageDetect_Exception + */ + public function detectConfidence($sample) + { + $scores = $this->detect($sample, 2); + + // if most similar language has the max score, it + // will have been picked at random + if (!is_array($scores) || empty($scores) + || current($scores) == $this->_max_score + ) { + return null; + } + + $arr['language'] = key($scores); + $arr['similarity'] = current($scores); + if (next($scores) !== false) { // if false then no next element + // the goal is to return a higher value if the distance between + // the similarity of the first score and the second score is high + + if ($this->_perl_compatible) { + $arr['confidence'] = (current($scores) - $arr['similarity']) + / $this->_max_score; + + } else { + $arr['confidence'] = $arr['similarity'] - current($scores); + + } + + } else { + $arr['confidence'] = null; + } + + return $arr; + } + + /** + * Returns the distribution of unicode blocks in a given utf8 string + * + * For the block name of a single char, use unicodeBlockName() + * + * @param string $str input string. Must be ascii or utf8 + * @param bool $skip_symbols if true, skip ascii digits, symbols and + * non-printing characters. Includes spaces, + * newlines and common punctutation characters. + * + * @return array + * @throws Text_LanguageDetect_Exception + */ + public function detectUnicodeBlocks($str, $skip_symbols) + { + $skip_symbols = (bool)$skip_symbols; + $str = (string)$str; + + $sample_obj = new Text_LanguageDetect_Parser($str); + $sample_obj->prepareUnicode(); + $sample_obj->prepareTrigram(false); + $sample_obj->setUnicodeSkipSymbols($skip_symbols); + $sample_obj->analyze(); + $blocks = $sample_obj->getUnicodeBlocks(); + unset($sample_obj); + return $blocks; + } + + /** + * Returns the block name for a given unicode value + * + * If passed a string, will assume it is being passed a UTF8-formatted + * character and will automatically convert. Otherwise it will assume it + * is being passed a numeric unicode value. + * + * Make sure input is of the correct type! + * + * @param mixed $unicode unicode value or utf8 char + * + * @return mixed the block name string or false if not found + * @throws Text_LanguageDetect_Exception + */ + public function unicodeBlockName($unicode) + { + if (is_string($unicode)) { + // assume it is being passed a utf8 char, so convert it + if (self::utf8strlen($unicode) > 1) { + throw new Text_LanguageDetect_Exception( + 'Pass a single char only to this method', + Text_LanguageDetect_Exception::PARAM_TYPE + ); + } + $unicode = $this->_utf8char2unicode($unicode); + + } elseif (!is_int($unicode)) { + throw new Text_LanguageDetect_Exception( + 'Input must be of type string or int.', + Text_LanguageDetect_Exception::PARAM_TYPE + ); + } + + $blocks = $this->_read_unicode_block_db(); + + $result = $this->_unicode_block_name($unicode, $blocks); + + if ($result == -1) { + return false; + } else { + return $result[2]; + } + } + + /** + * Searches the unicode block database + * + * Returns the block name for a given unicode value. unicodeBlockName() is + * the public interface for this function, which does input checks which + * this function omits for speed. + * + * @param int $unicode the unicode value + * @param array $blocks the block database + * @param int $block_count the number of defined blocks in the database + * + * @return mixed Block name, -1 if it failed + * @see unicodeBlockName() + * @access protected + */ + function _unicode_block_name($unicode, $blocks, $block_count = -1) + { + // for a reference, see + // http://www.unicode.org/Public/UNIDATA/Blocks.txt + + // assume that ascii characters are the most common + // so try it first for efficiency + if ($unicode <= $blocks[0][1]) { + return $blocks[0]; + } + + // the optional $block_count param is for efficiency + // so we this function doesn't have to run count() every time + if ($block_count != -1) { + $high = $block_count - 1; + } else { + $high = count($blocks) - 1; + } + + $low = 1; // start with 1 because ascii was 0 + + // your average binary search algorithm + while ($low <= $high) { + $mid = floor(($low + $high) / 2); + + if ($unicode < $blocks[$mid][0]) { + // if it's lower than the lower bound + $high = $mid - 1; + + } elseif ($unicode > $blocks[$mid][1]) { + // if it's higher than the upper bound + $low = $mid + 1; + + } else { + // found it + return $blocks[$mid]; + } + } + + // failed to find the block + return -1; + + // todo: differentiate when it's out of range or when it falls + // into an unassigned range? + } + + /** + * Brings up the unicode block database + * + * @return array the database of unicode block definitions + * @throws Text_LanguageDetect_Exception + * @access protected + */ + function _read_unicode_block_db() + { + // since the unicode definitions are always going to be the same, + // might as well share the memory for the db with all other instances + // of this class + static $data; + + if (!isset($data)) { + $data = $this->_readdb($this->_unicode_db_filename); + } + + return $data; + } + + /** + * Calculate the similarities between the language models + * + * Use this function to see how similar languages are to each other. + * + * If passed 2 language names, will return just those languages compared. + * If passed 1 language name, will return that language compared to + * all others. + * If passed none, will return an array of every language model compared + * to every other one. + * + * @param string $lang1 the name of the first language to be compared + * @param string $lang2 the name of the second language to be compared + * + * @return array scores of every language compared + * or the score of just the provided languages + * or null if one of the supplied languages does not exist + * @throws Text_LanguageDetect_Exception + */ + public function languageSimilarity($lang1 = null, $lang2 = null) + { + $lang1 = $this->_convertFromNameMode($lang1); + $lang2 = $this->_convertFromNameMode($lang2); + if ($lang1 != null) { + $lang1 = strtolower($lang1); + + // check if language model exists + if (!isset($this->_lang_db[$lang1])) { + return null; + } + + if ($lang2 != null) { + if (!isset($this->_lang_db[$lang2])) { + // check if language model exists + return null; + } + + $lang2 = strtolower($lang2); + + // compare just these two languages + return $this->_normalize_score( + $this->_distance( + $this->_lang_db[$lang1], + $this->_lang_db[$lang2] + ) + ); + + } else { + // compare just $lang1 to all languages + $return_arr = array(); + foreach ($this->_lang_db as $key => $value) { + if ($key != $lang1) { + // don't compare a language to itself + $return_arr[$key] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang1], $value) + ); + } + } + asort($return_arr); + + return $return_arr; + } + + + } else { + // compare all languages to each other + $return_arr = array(); + foreach (array_keys($this->_lang_db) as $lang1) { + foreach (array_keys($this->_lang_db) as $lang2) { + // skip comparing languages to themselves + if ($lang1 != $lang2) { + + if (isset($return_arr[$lang2][$lang1])) { + // don't re-calculate what's already been done + $return_arr[$lang1][$lang2] + = $return_arr[$lang2][$lang1]; + + } else { + // calculate + $return_arr[$lang1][$lang2] + = $this->_normalize_score( + $this->_distance( + $this->_lang_db[$lang1], + $this->_lang_db[$lang2] + ) + ); + + } + } + } + } + return $return_arr; + } + } + + /** + * Cluster known languages according to languageSimilarity() + * + * WARNING: this method is EXPERIMENTAL. It is not recommended for common + * use, and it may disappear or its functionality may change in future + * releases without notice. + * + * Uses a nearest neighbor technique to generate the maximum possible + * number of dendograms from the similarity data. + * + * @access public + * @return array language cluster data + * @throws Text_LanguageDetect_Exception + * @see languageSimilarity() + * @deprecated this function will eventually be removed and placed into + * the model generation class + */ + function clusterLanguages() + { + // todo: set the maximum number of clusters + // return cached result, if any + if (isset($this->_clusters)) { + return $this->_clusters; + } + + $langs = array_keys($this->_lang_db); + + $arr = $this->languageSimilarity(); + + sort($langs); + + foreach ($langs as $lang) { + if (!isset($this->_lang_db[$lang])) { + throw new Text_LanguageDetect_Exception( + "missing $lang!", + Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE + ); + } + } + + // http://www.psychstat.missouristate.edu/multibook/mlt04m.html + foreach ($langs as $old_key => $lang1) { + $langs[$lang1] = $lang1; + unset($langs[$old_key]); + } + + $result_data = $really_map = array(); + + $i = 0; + while (count($langs) > 2 && $i++ < 200) { + $highest_score = -1; + $highest_key1 = ''; + $highest_key2 = ''; + foreach ($langs as $lang1) { + foreach ($langs as $lang2) { + if ($lang1 != $lang2 + && $arr[$lang1][$lang2] > $highest_score + ) { + $highest_score = $arr[$lang1][$lang2]; + $highest_key1 = $lang1; + $highest_key2 = $lang2; + } + } + } + + if (!$highest_key1) { + // should not ever happen + throw new Text_LanguageDetect_Exception( + "no highest key? (step: $i)", + Text_LanguageDetect_Exception::NO_HIGHEST_KEY + ); + } + + if ($highest_score == 0) { + // languages are perfectly dissimilar + break; + } + + // $highest_key1 and $highest_key2 are most similar + $sum1 = array_sum($arr[$highest_key1]); + $sum2 = array_sum($arr[$highest_key2]); + + // use the score for the one that is most similar to the rest of + // the field as the score for the group + // todo: could try averaging or "centroid" method instead + // seems like that might make more sense + // actually nearest neighbor may be better for binary searching + + + // for "Complete Linkage"/"furthest neighbor" + // sign should be < + // for "Single Linkage"/"nearest neighbor" method + // should should be > + // results seem to be pretty much the same with either method + + // figure out which to delete and which to replace + if ($sum1 > $sum2) { + $replaceme = $highest_key1; + $deleteme = $highest_key2; + } else { + $replaceme = $highest_key2; + $deleteme = $highest_key1; + } + + $newkey = $replaceme . ':' . $deleteme; + + // $replaceme is most similar to remaining languages + // replace $replaceme with '$newkey', deleting $deleteme + + // keep a record of which fork is really which language + $really_lang = $replaceme; + while (isset($really_map[$really_lang])) { + $really_lang = $really_map[$really_lang]; + } + $really_map[$newkey] = $really_lang; + + + // replace the best fitting key, delete the other + foreach ($arr as $key1 => $arr2) { + foreach ($arr2 as $key2 => $value2) { + if ($key2 == $replaceme) { + $arr[$key1][$newkey] = $arr[$key1][$key2]; + unset($arr[$key1][$key2]); + // replacing $arr[$key1][$key2] with $arr[$key1][$newkey] + } + + if ($key1 == $replaceme) { + $arr[$newkey][$key2] = $arr[$key1][$key2]; + unset($arr[$key1][$key2]); + // replacing $arr[$key1][$key2] with $arr[$newkey][$key2] + } + + if ($key1 == $deleteme || $key2 == $deleteme) { + // deleting $arr[$key1][$key2] + unset($arr[$key1][$key2]); + } + } + } + + + unset($langs[$highest_key1]); + unset($langs[$highest_key2]); + $langs[$newkey] = $newkey; + + + // some of these may be overkill + $result_data[$newkey] = array( + 'newkey' => $newkey, + 'count' => $i, + 'diff' => abs($sum1 - $sum2), + 'score' => $highest_score, + 'bestfit' => $replaceme, + 'otherfit' => $deleteme, + 'really' => $really_lang, + ); + } + + $return_val = array( + 'open_forks' => $langs, + // the top level of clusters + // clusters that are mutually exclusive + // or specified by a specific maximum + + 'fork_data' => $result_data, + // data for each split + + 'name_map' => $really_map, + // which cluster is really which language + // using the nearest neighbor technique, the cluster + // inherits all of the properties of its most-similar member + // this keeps track + ); + + + // saves the result in the object + $this->_clusters = $return_val; + + return $return_val; + } + + + /** + * Perform an intelligent detection based on clusterLanguages() + * + * WARNING: this method is EXPERIMENTAL. It is not recommended for common + * use, and it may disappear or its functionality may change in future + * releases without notice. + * + * This compares the sample text to top the top level of clusters. If the + * sample is similar to the cluster it will drop down and compare it to the + * languages in the cluster, and so on until it hits a leaf node. + * + * this should find the language in considerably fewer compares + * (the equivalent of a binary search), however clusterLanguages() is costly + * and the loss of accuracy from this technique is significant. + * + * This method may need to be 'fuzzier' in order to become more accurate. + * + * This function could be more useful if the universe of possible languages + * was very large, however in such cases some method of Bayesian inference + * might be more helpful. + * + * @param string $str input string + * + * @return array language scores (only those compared) + * @throws Text_LanguageDetect_Exception + * @see clusterLanguages() + */ + public function clusteredSearch($str) + { + // input check + if (!Text_LanguageDetect_Parser::validateString($str)) { + return array(); + } + + // clusterLanguages() will return a cached result if possible + // so it's safe to call it every time + $result = $this->clusterLanguages(); + + $dendogram_start = $result['open_forks']; + $dendogram_data = $result['fork_data']; + $dendogram_alias = $result['name_map']; + + $sample_obj = new Text_LanguageDetect_Parser($str); + $sample_obj->prepareTrigram(); + $sample_obj->setPadStart(!$this->_perl_compatible); + $sample_obj->analyze(); + $sample_result = $sample_obj->getTrigramRanks(); + $sample_count = count($sample_result); + + // input check + if ($sample_count == 0) { + return array(); + } + + $i = 0; // counts the number of steps + + foreach ($dendogram_start as $lang) { + if (isset($dendogram_alias[$lang])) { + $lang_key = $dendogram_alias[$lang]; + } else { + $lang_key = $lang; + } + + $scores[$lang] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang_key], $sample_result), + $sample_count + ); + + $i++; + } + + if ($this->_perl_compatible) { + asort($scores); + } else { + arsort($scores); + } + + $top_score = current($scores); + $top_key = key($scores); + + // of starting forks, $top_key is the most similar to the sample + + $cur_key = $top_key; + while (isset($dendogram_data[$cur_key])) { + $lang1 = $dendogram_data[$cur_key]['bestfit']; + $lang2 = $dendogram_data[$cur_key]['otherfit']; + foreach (array($lang1, $lang2) as $lang) { + if (isset($dendogram_alias[$lang])) { + $lang_key = $dendogram_alias[$lang]; + } else { + $lang_key = $lang; + } + + $scores[$lang] = $this->_normalize_score( + $this->_distance($this->_lang_db[$lang_key], $sample_result), + $sample_count + ); + + //todo: does not need to do same comparison again + } + + $i++; + + if ($scores[$lang1] > $scores[$lang2]) { + $cur_key = $lang1; + $loser_key = $lang2; + } else { + $cur_key = $lang2; + $loser_key = $lang1; + } + + $diff = $scores[$cur_key] - $scores[$loser_key]; + + // $cur_key ({$dendogram_alias[$cur_key]}) wins + // over $loser_key ({$dendogram_alias[$loser_key]}) + // with a difference of $diff + } + + // found result in $i compares + + // rather than sorting the result, preserve it so that you can see + // which paths the algorithm decided to take along the tree + + // but sometimes the last item is only the second highest + if (($this->_perl_compatible && (end($scores) > prev($scores))) + || (!$this->_perl_compatible && (end($scores) < prev($scores))) + ) { + $real_last_score = current($scores); + $real_last_key = key($scores); + + // swaps the 2nd-to-last item for the last item + unset($scores[$real_last_key]); + $scores[$real_last_key] = $real_last_score; + } + + + if (!$this->_perl_compatible) { + $scores = array_reverse($scores, true); + // second param requires php > 4.0.3 + } + + return $scores; + } + + /** + * ut8-safe strlen() + * + * Returns the numbers of characters (not bytes) in a utf8 string + * + * @param string $str string to get the length of + * + * @return int number of chars + */ + public static function utf8strlen($str) + { + // utf8_decode() will convert unknown chars to '?', which is actually + // ideal for counting. + + return strlen(utf8_decode($str)); + + // idea stolen from dokuwiki + } + + /** + * Returns the unicode value of a utf8 char + * + * @param string $char a utf8 (possibly multi-byte) char + * + * @return int unicode value + * @access protected + * @link http://en.wikipedia.org/wiki/UTF-8 + */ + function _utf8char2unicode($char) + { + // strlen() here will actually get the binary length of a single char + switch (strlen($char)) { + case 1: + // normal ASCII-7 byte + // 0xxxxxxx --> 0xxxxxxx + return ord($char{0}); + + case 2: + // 2 byte unicode + // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx + $z = (ord($char{0}) & 0x000001F) << 6; + $x = (ord($char{1}) & 0x0000003F); + return ($z | $x); + + case 3: + // 3 byte unicode + // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx + $z = (ord($char{0}) & 0x0000000F) << 12; + $x1 = (ord($char{1}) & 0x0000003F) << 6; + $x2 = (ord($char{2}) & 0x0000003F); + return ($z | $x1 | $x2); + + case 4: + // 4 byte unicode + // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx --> + // 000zzzzz xxxxxxxx xxxxxxxx + $z1 = (ord($char{0}) & 0x00000007) << 18; + $z2 = (ord($char{1}) & 0x0000003F) << 12; + $x1 = (ord($char{2}) & 0x0000003F) << 6; + $x2 = (ord($char{3}) & 0x0000003F); + return ($z1 | $z2 | $x1 | $x2); + } + } + + /** + * utf8-safe fast character iterator + * + * Will get the next character starting from $counter, which will then be + * incremented. If a multi-byte char the bytes will be concatenated and + * $counter will be incremeted by the number of bytes in the char. + * + * @param string $str the string being iterated over + * @param int &$counter the iterator, will increment by reference + * @param bool $special_convert whether to do special conversions + * + * @return char the next (possibly multi-byte) char from $counter + * @access private + */ + static function _next_char($str, &$counter, $special_convert = false) + { + $char = $str{$counter++}; + $ord = ord($char); + + // for a description of the utf8 system see + // http://www.phpclasses.org/browse/file/5131.html + + // normal ascii one byte char + if ($ord <= 127) { + // special conversions needed for this package + // (that only apply to regular ascii characters) + // lower case, and convert all non-alphanumeric characters + // other than "'" to space + if ($special_convert && $char != ' ' && $char != "'") { + if ($ord >= 65 && $ord <= 90) { // A-Z + $char = chr($ord + 32); // lower case + } elseif ($ord < 97 || $ord > 122) { // NOT a-z + $char = ' '; // convert to space + } + } + + return $char; + + } elseif ($ord >> 5 == 6) { // two-byte char + // multi-byte chars + $nextchar = $str{$counter++}; // get next byte + + // lower-casing of non-ascii characters is still incomplete + + if ($special_convert) { + // lower case latin accented characters + if ($ord == 195) { + $nextord = ord($nextchar); + $nextord_adj = $nextord + 64; + // for a reference, see + // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html + + // À - Þ but not × + if ($nextord_adj >= 192 + && $nextord_adj <= 222 + && $nextord_adj != 215 + ) { + $nextchar = chr($nextord + 32); + } + + } elseif ($ord == 208) { + // lower case cyrillic alphabet + $nextord = ord($nextchar); + // if A - Pe + if ($nextord >= 144 && $nextord <= 159) { + // lower case + $nextchar = chr($nextord + 32); + + } elseif ($nextord >= 160 && $nextord <= 175) { + // if Er - Ya + // lower case + $char = chr(209); // == $ord++ + $nextchar = chr($nextord - 32); + } + } + } + + // tag on next byte + return $char . $nextchar; + } elseif ($ord >> 4 == 14) { // three-byte char + + // tag on next 2 bytes + return $char . $str{$counter++} . $str{$counter++}; + + } elseif ($ord >> 3 == 30) { // four-byte char + + // tag on next 3 bytes + return $char . $str{$counter++} . $str{$counter++} . $str{$counter++}; + + } else { + // error? + } + } + + /** + * Converts an $language input parameter from the configured mode + * to the language name that is used internally. + * + * Works for strings and arrays. + * + * @param string|array $lang A language description ("english"/"en"/"eng") + * @param boolean $convertKey If $lang is an array, setting $key + * converts the keys to the language name. + * + * @return string|array Language name + */ + function _convertFromNameMode($lang, $convertKey = false) + { + if ($this->_name_mode == 0) { + return $lang; + } + + if ($this->_name_mode == 2) { + $method = 'code2ToName'; + } else { + $method = 'code3ToName'; + } + + if (is_string($lang)) { + return (string)Text_LanguageDetect_ISO639::$method($lang); + } + + $newlang = array(); + foreach ($lang as $key => $val) { + if ($convertKey) { + $newkey = (string)Text_LanguageDetect_ISO639::$method($key); + $newlang[$newkey] = $val; + } else { + $newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val); + } + } + return $newlang; + } + + /** + * Converts an $language output parameter from the language name that is + * used internally to the configured mode. + * + * Works for strings and arrays. + * + * @param string|array $lang A language description ("english"/"en"/"eng") + * @param boolean $convertKey If $lang is an array, setting $key + * converts the keys to the language name. + * + * @return string|array Language name + */ + function _convertToNameMode($lang, $convertKey = false) + { + if ($this->_name_mode == 0) { + return $lang; + } + + if ($this->_name_mode == 2) { + $method = 'nameToCode2'; + } else { + $method = 'nameToCode3'; + } + + if (is_string($lang)) { + return Text_LanguageDetect_ISO639::$method($lang); + } + + $newlang = array(); + foreach ($lang as $key => $val) { + if ($convertKey) { + $newkey = Text_LanguageDetect_ISO639::$method($key); + $newlang[$newkey] = $val; + } else { + $newlang[$key] = Text_LanguageDetect_ISO639::$method($val); + } + } + return $newlang; + } +} + +/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */ + +?> diff --git a/library/langdet/Text/LanguageDetect/Exception.php b/library/langdet/Text/LanguageDetect/Exception.php new file mode 100644 index 000000000..196d994f5 --- /dev/null +++ b/library/langdet/Text/LanguageDetect/Exception.php @@ -0,0 +1,57 @@ +<?php +class Text_LanguageDetect_Exception extends Exception +{ + /** + * Database file could not be found + */ + const DB_NOT_FOUND = 10; + + /** + * Database file found, but not readable + */ + const DB_NOT_READABLE = 11; + + /** + * Database file is empty + */ + const DB_EMPTY = 12; + + /** + * Database contents is not a PHP array + */ + const DB_NOT_ARRAY = 13; + + /** + * Magic quotes are activated + */ + const MAGIC_QUOTES = 14; + + + /** + * Parameter of invalid type passed to method + */ + const PARAM_TYPE = 20; + + /** + * Character in parameter is invalid + */ + const INVALID_CHAR = 21; + + + /** + * Language is not in the database + */ + const UNKNOWN_LANGUAGE = 30; + + + /** + * Error during block detection + */ + const BLOCK_DETECTION = 40; + + + /** + * Error while clustering languages + */ + const NO_HIGHEST_KEY = 50; +} diff --git a/library/langdet/Text/LanguageDetect/ISO639.php b/library/langdet/Text/LanguageDetect/ISO639.php new file mode 100644 index 000000000..c577a2e1a --- /dev/null +++ b/library/langdet/Text/LanguageDetect/ISO639.php @@ -0,0 +1,341 @@ +<?php +/** + * Part of Text_LanguageDetect + * + * PHP version 5 + * + * @category Text + * @package Text_LanguageDetect + * @author Christian Weiske <cweiske@php.net> + * @copyright 2011 Christian Weiske <cweiske@php.net> + * @license http://www.debian.org/misc/bsd.license BSD + * @version SVN: $Id$ + * @link http://pear.php.net/package/Text_LanguageDetect/ + */ + +/** + * Provides a mapping between the languages from lang.dat and the + * ISO 639-1 and ISO-639-2 codes. + * + * Note that this class contains only languages that exist in lang.dat. + * + * @category Text + * @package Text_LanguageDetect + * @author Christian Weiske <cweiske@php.net> + * @copyright 2011 Christian Weiske <cweiske@php.net> + * @license http://www.debian.org/misc/bsd.license BSD + * @link http://www.loc.gov/standards/iso639-2/php/code_list.php + */ +class Text_LanguageDetect_ISO639 +{ + /** + * Maps all language names from the language database to the + * ISO 639-1 2-letter language code. + * + * NULL indicates that there is no 2-letter code. + * + * @var array + */ + public static $nameToCode2 = array( + 'albanian' => 'sq', + 'arabic' => 'ar', + 'azeri' => 'az', + 'bengali' => 'bn', + 'bulgarian' => 'bg', + 'cebuano' => null, + 'croatian' => 'hr', + 'czech' => 'cs', + 'danish' => 'da', + 'dutch' => 'nl', + 'english' => 'en', + 'estonian' => 'et', + 'farsi' => 'fa', + 'finnish' => 'fi', + 'french' => 'fr', + 'german' => 'de', + 'hausa' => 'ha', + 'hawaiian' => null, + 'hindi' => 'hi', + 'hungarian' => 'hu', + 'icelandic' => 'is', + 'indonesian' => 'id', + 'italian' => 'it', + 'kazakh' => 'kk', + 'kyrgyz' => 'ky', + 'latin' => 'la', + 'latvian' => 'lv', + 'lithuanian' => 'lt', + 'macedonian' => 'mk', + 'mongolian' => 'mn', + 'nepali' => 'ne', + 'norwegian' => 'no', + 'pashto' => 'ps', + 'pidgin' => null, + 'polish' => 'pl', + 'portuguese' => 'pt', + 'romanian' => 'ro', + 'russian' => 'ru', + 'serbian' => 'sr', + 'slovak' => 'sk', + 'slovene' => 'sl', + 'somali' => 'so', + 'spanish' => 'es', + 'swahili' => 'sw', + 'swedish' => 'sv', + 'tagalog' => 'tl', + 'turkish' => 'tr', + 'ukrainian' => 'uk', + 'urdu' => 'ur', + 'uzbek' => 'uz', + 'vietnamese' => 'vi', + 'welsh' => 'cy', + ); + + /** + * Maps all language names from the language database to the + * ISO 639-2 3-letter language code. + * + * @var array + */ + public static $nameToCode3 = array( + 'albanian' => 'sqi', + 'arabic' => 'ara', + 'azeri' => 'aze', + 'bengali' => 'ben', + 'bulgarian' => 'bul', + 'cebuano' => 'ceb', + 'croatian' => 'hrv', + 'czech' => 'ces', + 'danish' => 'dan', + 'dutch' => 'nld', + 'english' => 'eng', + 'estonian' => 'est', + 'farsi' => 'fas', + 'finnish' => 'fin', + 'french' => 'fra', + 'german' => 'deu', + 'hausa' => 'hau', + 'hawaiian' => 'haw', + 'hindi' => 'hin', + 'hungarian' => 'hun', + 'icelandic' => 'isl', + 'indonesian' => 'ind', + 'italian' => 'ita', + 'kazakh' => 'kaz', + 'kyrgyz' => 'kir', + 'latin' => 'lat', + 'latvian' => 'lav', + 'lithuanian' => 'lit', + 'macedonian' => 'mkd', + 'mongolian' => 'mon', + 'nepali' => 'nep', + 'norwegian' => 'nor', + 'pashto' => 'pus', + 'pidgin' => 'crp', + 'polish' => 'pol', + 'portuguese' => 'por', + 'romanian' => 'ron', + 'russian' => 'rus', + 'serbian' => 'srp', + 'slovak' => 'slk', + 'slovene' => 'slv', + 'somali' => 'som', + 'spanish' => 'spa', + 'swahili' => 'swa', + 'swedish' => 'swe', + 'tagalog' => 'tgl', + 'turkish' => 'tur', + 'ukrainian' => 'ukr', + 'urdu' => 'urd', + 'uzbek' => 'uzb', + 'vietnamese' => 'vie', + 'welsh' => 'cym', + ); + + /** + * Maps ISO 639-1 2-letter language codes to the language names + * in the language database + * + * Not all languages have a 2 letter code, so some are missing + * + * @var array + */ + public static $code2ToName = array( + 'ar' => 'arabic', + 'az' => 'azeri', + 'bg' => 'bulgarian', + 'bn' => 'bengali', + 'cs' => 'czech', + 'cy' => 'welsh', + 'da' => 'danish', + 'de' => 'german', + 'en' => 'english', + 'es' => 'spanish', + 'et' => 'estonian', + 'fa' => 'farsi', + 'fi' => 'finnish', + 'fr' => 'french', + 'ha' => 'hausa', + 'hi' => 'hindi', + 'hr' => 'croatian', + 'hu' => 'hungarian', + 'id' => 'indonesian', + 'is' => 'icelandic', + 'it' => 'italian', + 'kk' => 'kazakh', + 'ky' => 'kyrgyz', + 'la' => 'latin', + 'lt' => 'lithuanian', + 'lv' => 'latvian', + 'mk' => 'macedonian', + 'mn' => 'mongolian', + 'ne' => 'nepali', + 'nl' => 'dutch', + 'no' => 'norwegian', + 'pl' => 'polish', + 'ps' => 'pashto', + 'pt' => 'portuguese', + 'ro' => 'romanian', + 'ru' => 'russian', + 'sk' => 'slovak', + 'sl' => 'slovene', + 'so' => 'somali', + 'sq' => 'albanian', + 'sr' => 'serbian', + 'sv' => 'swedish', + 'sw' => 'swahili', + 'tl' => 'tagalog', + 'tr' => 'turkish', + 'uk' => 'ukrainian', + 'ur' => 'urdu', + 'uz' => 'uzbek', + 'vi' => 'vietnamese', + ); + + /** + * Maps ISO 639-2 3-letter language codes to the language names + * in the language database. + * + * @var array + */ + public static $code3ToName = array( + 'ara' => 'arabic', + 'aze' => 'azeri', + 'ben' => 'bengali', + 'bul' => 'bulgarian', + 'ceb' => 'cebuano', + 'ces' => 'czech', + 'crp' => 'pidgin', + 'cym' => 'welsh', + 'dan' => 'danish', + 'deu' => 'german', + 'eng' => 'english', + 'est' => 'estonian', + 'fas' => 'farsi', + 'fin' => 'finnish', + 'fra' => 'french', + 'hau' => 'hausa', + 'haw' => 'hawaiian', + 'hin' => 'hindi', + 'hrv' => 'croatian', + 'hun' => 'hungarian', + 'ind' => 'indonesian', + 'isl' => 'icelandic', + 'ita' => 'italian', + 'kaz' => 'kazakh', + 'kir' => 'kyrgyz', + 'lat' => 'latin', + 'lav' => 'latvian', + 'lit' => 'lithuanian', + 'mkd' => 'macedonian', + 'mon' => 'mongolian', + 'nep' => 'nepali', + 'nld' => 'dutch', + 'nor' => 'norwegian', + 'pol' => 'polish', + 'por' => 'portuguese', + 'pus' => 'pashto', + 'rom' => 'romanian', + 'rus' => 'russian', + 'slk' => 'slovak', + 'slv' => 'slovene', + 'som' => 'somali', + 'spa' => 'spanish', + 'sqi' => 'albanian', + 'srp' => 'serbian', + 'swa' => 'swahili', + 'swe' => 'swedish', + 'tgl' => 'tagalog', + 'tur' => 'turkish', + 'ukr' => 'ukrainian', + 'urd' => 'urdu', + 'uzb' => 'uzbek', + 'vie' => 'vietnamese', + ); + + /** + * Returns the 2-letter ISO 639-1 code for the given language name. + * + * @param string $lang English language name like "swedish" + * + * @return string Two-letter language code (e.g. "sv") or NULL if not found + */ + public static function nameToCode2($lang) + { + $lang = strtolower($lang); + if (!isset(self::$nameToCode2[$lang])) { + return null; + } + return self::$nameToCode2[$lang]; + } + + /** + * Returns the 3-letter ISO 639-2 code for the given language name. + * + * @param string $lang English language name like "swedish" + * + * @return string Three-letter language code (e.g. "swe") or NULL if not found + */ + public static function nameToCode3($lang) + { + $lang = strtolower($lang); + if (!isset(self::$nameToCode3[$lang])) { + return null; + } + return self::$nameToCode3[$lang]; + } + + /** + * Returns the language name for the given 2-letter ISO 639-1 code. + * + * @param string $code Two-letter language code (e.g. "sv") + * + * @return string English language name like "swedish" + */ + public static function code2ToName($code) + { + $lang = strtolower($code); + if (!isset(self::$code2ToName[$code])) { + return null; + } + return self::$code2ToName[$code]; + } + + /** + * Returns the language name for the given 3-letter ISO 639-2 code. + * + * @param string $code Three-letter language code (e.g. "swe") + * + * @return string English language name like "swedish" + */ + public static function code3ToName($code) + { + $lang = strtolower($code); + if (!isset(self::$code3ToName[$code])) { + return null; + } + return self::$code3ToName[$code]; + } +} + +?>
\ No newline at end of file diff --git a/library/langdet/Text/LanguageDetect/Parser.php b/library/langdet/Text/LanguageDetect/Parser.php new file mode 100644 index 000000000..1c20c2657 --- /dev/null +++ b/library/langdet/Text/LanguageDetect/Parser.php @@ -0,0 +1,349 @@ +<?php + +/** + * This class represents a text sample to be parsed. + * + * @category Text + * @package Text_LanguageDetect + * @author Nicholas Pisarro + * @copyright 2006 + * @license BSD + * @version CVS: $Id: Parser.php 322327 2012-01-15 17:55:59Z cweiske $ + * @link http://pear.php.net/package/Text_LanguageDetect/ + * @link http://langdetect.blogspot.com/ + */ + +/** + * This class represents a text sample to be parsed. + * + * This separates the analysis of a text sample from the primary LanguageDetect + * class. After a new profile has been built, the data can be retrieved using + * the accessor functions. + * + * This class is intended to be used by the Text_LanguageDetect class, not + * end-users. + * + * @category Text + * @package Text_LanguageDetect + * @author Nicholas Pisarro + * @copyright 2006 + * @license BSD + * @version release: 0.3.0 + */ +class Text_LanguageDetect_Parser extends Text_LanguageDetect +{ + /** + * the piece of text being parsed + * + * @access private + * @var string + */ + var $_string; + + /** + * stores the trigram frequencies of the sample + * + * @access private + * @var string + */ + var $_trigrams = array(); + + /** + * stores the trigram ranks of the sample + * + * @access private + * @var array + */ + var $_trigram_ranks = array(); + + /** + * stores the unicode blocks of the sample + * + * @access private + * @var array + */ + var $_unicode_blocks = array(); + + /** + * Whether the parser should compile the unicode ranges + * + * @access private + * @var bool + */ + var $_compile_unicode = false; + + /** + * Whether the parser should compile trigrams + * + * @access private + * @var bool + */ + var $_compile_trigram = false; + + /** + * Whether the trigram parser should pad the beginning of the string + * + * @access private + * @var bool + */ + var $_trigram_pad_start = false; + + /** + * Whether the unicode parser should skip non-alphabetical ascii chars + * + * @access private + * @var bool + */ + var $_unicode_skip_symbols = true; + + /** + * Constructor + * + * @access private + * @param string $string string to be parsed + */ + function Text_LanguageDetect_Parser($string) { + $this->_string = $string; + } + + /** + * Returns true if a string is suitable for parsing + * + * @param string $str input string to test + * @return bool true if acceptable, false if not + */ + public static function validateString($str) { + if (!empty($str) && strlen($str) > 3 && preg_match('/\S/', $str)) { + return true; + } else { + return false; + } + } + + /** + * turn on/off trigram counting + * + * @access public + * @param bool $bool true for on, false for off + */ + function prepareTrigram($bool = true) + { + $this->_compile_trigram = $bool; + } + + /** + * turn on/off unicode block counting + * + * @access public + * @param bool $bool true for on, false for off + */ + function prepareUnicode($bool = true) + { + $this->_compile_unicode = $bool; + } + + /** + * turn on/off padding the beginning of the sample string + * + * @access public + * @param bool $bool true for on, false for off + */ + function setPadStart($bool = true) + { + $this->_trigram_pad_start = $bool; + } + + /** + * Should the unicode block counter skip non-alphabetical ascii chars? + * + * @access public + * @param bool $bool true for on, false for off + */ + function setUnicodeSkipSymbols($bool = true) + { + $this->_unicode_skip_symbols = $bool; + } + + /** + * Returns the trigram ranks for the text sample + * + * @access public + * @return array trigram ranks in the text sample + */ + function &getTrigramRanks() + { + return $this->_trigram_ranks; + } + + /** + * Return the trigram freqency table + * + * only used in testing to make sure the parser is working + * + * @access public + * @return array trigram freqencies in the text sample + */ + function &getTrigramFreqs() + { + return $this->_trigram; + } + + /** + * returns the array of unicode blocks + * + * @access public + * @return array unicode blocks in the text sample + */ + function &getUnicodeBlocks() + { + return $this->_unicode_blocks; + } + + /** + * Executes the parsing operation + * + * Be sure to call the set*() functions to set options and the + * prepare*() functions first to tell it what kind of data to compute + * + * Afterwards the get*() functions can be used to access the compiled + * information. + * + * @access public + */ + function analyze() + { + $len = strlen($this->_string); + $byte_counter = 0; + + + // unicode startup + if ($this->_compile_unicode) { + $blocks = $this->_read_unicode_block_db(); + $block_count = count($blocks); + + $skipped_count = 0; + $unicode_chars = array(); + } + + // trigram startup + if ($this->_compile_trigram) { + // initialize them as blank so the parser will skip the first two + // (since it skips trigrams with more than 2 contiguous spaces) + $a = ' '; + $b = ' '; + + // kludge + // if it finds a valid trigram to start and the start pad option is + // off, then set a variable that will be used to reduce this + // trigram after parsing has finished + if (!$this->_trigram_pad_start) { + $a = $this->_next_char($this->_string, $byte_counter, true); + + if ($a != ' ') { + $b = $this->_next_char($this->_string, $byte_counter, true); + $dropone = " $a$b"; + } + + $byte_counter = 0; + $a = ' '; + $b = ' '; + } + } + + while ($byte_counter < $len) { + $char = $this->_next_char($this->_string, $byte_counter, true); + + + // language trigram detection + if ($this->_compile_trigram) { + if (!($b == ' ' && ($a == ' ' || $char == ' '))) { + if (!isset($this->_trigram[$a . $b . $char])) { + $this->_trigram[$a . $b . $char] = 1; + } else { + $this->_trigram[$a . $b . $char]++; + } + } + + $a = $b; + $b = $char; + } + + // unicode block detection + if ($this->_compile_unicode) { + if ($this->_unicode_skip_symbols + && strlen($char) == 1 + && ($char < 'A' || $char > 'z' + || ($char > 'Z' && $char < 'a')) + && $char != "'") { // does not skip the apostrophe + // since it's included in the language + // models + + $skipped_count++; + continue; + } + + // build an array of all the characters + if (isset($unicode_chars[$char])) { + $unicode_chars[$char]++; + } else { + $unicode_chars[$char] = 1; + } + } + + // todo: add byte detection here + } + + // unicode cleanup + if ($this->_compile_unicode) { + foreach ($unicode_chars as $utf8_char => $count) { + $search_result = $this->_unicode_block_name( + $this->_utf8char2unicode($utf8_char), $blocks, $block_count); + + if ($search_result != -1) { + $block_name = $search_result[2]; + } else { + $block_name = '[Malformatted]'; + } + + if (isset($this->_unicode_blocks[$block_name])) { + $this->_unicode_blocks[$block_name] += $count; + } else { + $this->_unicode_blocks[$block_name] = $count; + } + } + } + + + // trigram cleanup + if ($this->_compile_trigram) { + // pad the end + if ($b != ' ') { + if (!isset($this->_trigram["$a$b "])) { + $this->_trigram["$a$b "] = 1; + } else { + $this->_trigram["$a$b "]++; + } + } + + // perl compatibility; Language::Guess does not pad the beginning + // kludge + if (isset($dropone)) { + if ($this->_trigram[$dropone] == 1) { + unset($this->_trigram[$dropone]); + } else { + $this->_trigram[$dropone]--; + } + } + + if (!empty($this->_trigram)) { + $this->_trigram_ranks = $this->_arr_rank($this->_trigram); + } else { + $this->_trigram_ranks = array(); + } + } + } +} + +/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */ + +?> |