Merge branch 'dev' of https://github.com/redmatrix/hubzilla into xdev_merge

4 files changed, 0 insertions, 2455 deletions

diff --git a/library/langdet/Text/LanguageDetect.php b/library/langdet/Text/LanguageDetect.php
deleted file mode 100644
index 7cebbe607..000000000
--- a/library/langdet/Text/LanguageDetect.php
+++ /dev/null
@@ -1,1708 +0,0 @@
-<?php
-
-/**
- * Detects the language of a given piece of text.
- *
- * Attempts to detect the language of a sample of text by correlating ranked
- * 3-gram frequencies to a table of 3-gram frequencies of known languages.
- *
- * Implements a version of a technique originally proposed by Cavnar & Trenkle
- * (1994): "N-Gram-Based Text Categorization"
- *
- * PHP version 5
- *
- * @category  Text
- * @package   Text_LanguageDetect
- * @author    Nicholas Pisarro <infinityminusnine+pear@gmail.com>
- * @copyright 2005-2006 Nicholas Pisarro
- * @license   http://www.debian.org/misc/bsd.license BSD
- * @version   SVN: $Id: LanguageDetect.php 322353 2012-01-16 08:41:43Z cweiske $
- * @link      http://pear.php.net/package/Text_LanguageDetect/
- * @link      http://langdetect.blogspot.com/
- */
-
-require_once 'Text/LanguageDetect/Exception.php';
-require_once 'Text/LanguageDetect/Parser.php';
-require_once 'Text/LanguageDetect/ISO639.php';
-
-/**
- * Language detection class
- *
- * Requires the langauge model database (lang.dat) that should have
- * accompanied this class definition in order to be instantiated.
- *
- * Example usage:
- *
- * <code>
- * require_once 'Text/LanguageDetect.php';
- *
- * $l = new Text_LanguageDetect;
- *
- * $stdin = fopen('php://stdin', 'r');
- *
- * echo "Supported languages:\n";
- *
- * try {
- *     $langs = $l->getLanguages();
- * } catch (Text_LanguageDetect_Exception $e) {
- *     die($e->getMessage());
- * }
- *
- * sort($langs);
- * echo join(', ', $langs);
- *
- * while ($line = fgets($stdin)) {
- *     print_r($l->detect($line, 4));
- * }
- * </code>
- *
- * @category  Text
- * @package   Text_LanguageDetect
- * @author    Nicholas Pisarro <infinityminusnine+pear@gmail.com>
- * @copyright 2005 Nicholas Pisarro
- * @license   http://www.debian.org/misc/bsd.license BSD
- * @version   Release: @package_version@
- * @link      http://pear.php.net/package/Text_LanguageDetect/
- * @todo      allow users to generate their own language models
- */
-class Text_LanguageDetect
-{
-    /**
-     * The filename that stores the trigram data for the detector
-     *
-     * If this value starts with a slash (/) or a dot (.) the value of
-     * $this->_data_dir will be ignored
-     *
-     * @var      string
-     * @access   private
-     */
-    var $_db_filename = 'lang.dat';
-
-    /**
-     * The filename that stores the unicode block definitions
-     *
-     * If this value starts with a slash (/) or a dot (.) the value of
-     * $this->_data_dir will be ignored
-     *
-     * @var string
-     * @access private
-     */
-    var $_unicode_db_filename = 'unicode_blocks.dat';
-
-    /**
-     * The data directory
-     *
-     * Should be set by PEAR installer
-     *
-     * @var      string
-     * @access   private
-     */
-    var $_data_dir = '@data_dir@';
-
-    /**
-     * The trigram data for comparison
-     *
-     * Will be loaded on start from $this->_db_filename
-     *
-     * @var      array
-     * @access   private
-     */
-    var $_lang_db = array();
-
-    /**
-     * stores the map of the trigram data to unicode characters
-     *
-     * @access private
-     * @var array
-     */
-    var $_unicode_map;
-
-    /**
-     * The size of the trigram data arrays
-     *
-     * @var      int
-     * @access   private
-     */
-    var $_threshold = 300;
-
-    /**
-     * the maximum possible score.
-     *
-     * needed for score normalization. Different depending on the
-     * perl compatibility setting
-     *
-     * @access  private
-     * @var     int
-     * @see     setPerlCompatible()
-     */
-    var $_max_score = 0;
-
-    /**
-     * Whether or not to simulate perl's Language::Guess exactly
-     *
-     * @access  private
-     * @var     bool
-     * @see     setPerlCompatible()
-     */
-    var $_perl_compatible = false;
-
-    /**
-     * Whether to use the unicode block detection to speed up processing
-     *
-     * @access private
-     * @var bool
-     */
-    var $_use_unicode_narrowing = true;
-
-    /**
-     * stores the result of the clustering operation
-     *
-     * @access  private
-     * @var     array
-     * @see     clusterLanguages()
-     */
-    var $_clusters;
-
-    /**
-     * Which type of "language names" are accepted and returned:
-     *
-     * 0 - language name ("english")
-     * 2 - 2-letter ISO 639-1 code ("en")
-     * 3 - 3-letter ISO 639-2 code ("eng")
-     */
-    var $_name_mode = 0;
-
-    /**
-     * Constructor
-     *
-     * Will attempt to load the language database. If it fails, you will get
-     * an exception.
-     */
-    function __construct()
-    {
-        $data = $this->_readdb($this->_db_filename);
-        $this->_checkTrigram($data['trigram']);
-        $this->_lang_db = $data['trigram'];
-
-        if (isset($data['trigram-unicodemap'])) {
-            $this->_unicode_map = $data['trigram-unicodemap'];
-        }
-
-        // Not yet implemented:
-        if (isset($data['trigram-clusters'])) {
-            $this->_clusters = $data['trigram-clusters'];
-        }
-    }
-
-    /**
-     * Returns the path to the location of the database
-     *
-     * @param string $fname File name to load
-     *
-     * @return string expected path to the language model database
-     * @access private
-     */
-    function _get_data_loc($fname)
-    {
-        if ($fname{0} == '/' || $fname{0} == '.') {
-            // if filename starts with a slash, assume it's an absolute pathname
-            // and skip whatever is in $this->_data_dir
-            return $fname;
-
-        } elseif ($this->_data_dir != '@' . 'data_dir' . '@') {
-            // if the data dir was set by the PEAR installer, use that
-            return $this->_data_dir . '/Text_LanguageDetect/' . $fname;
-
-        } else {
-            // assume this was just unpacked somewhere
-            // try the local working directory if otherwise
-            return __DIR__ . '/../data/' . $fname;
-        }
-    }
-
-    /**
-     * Loads the language trigram database from filename
-     *
-     * Trigram datbase should be a serialize()'d array
-     *
-     * @param string $fname the filename where the data is stored
-     *
-     * @return array the language model data
-     * @throws Text_LanguageDetect_Exception
-     * @access private
-     */
-    function _readdb($fname)
-    {
-        // finds the correct data dir
-        $fname = $this->_get_data_loc($fname);
-
-        // input check
-        if (!file_exists($fname)) {
-            throw new Text_LanguageDetect_Exception(
-                'Language database does not exist: ' . $fname,
-                Text_LanguageDetect_Exception::DB_NOT_FOUND
-            );
-        } elseif (!is_readable($fname)) {
-            throw new Text_LanguageDetect_Exception(
-                'Language database is not readable: ' . $fname,
-                Text_LanguageDetect_Exception::DB_NOT_READABLE
-            );
-        }
-
-        return unserialize(file_get_contents($fname));
-    }
-
-
-    /**
-     * Checks if this object is ready to detect languages
-     *
-     * @param array $trigram Trigram data from database
-     *
-     * @return void
-     * @access private
-     */
-    function _checkTrigram($trigram)
-    {
-        if (!is_array($trigram)) {
-            if (ini_get('magic_quotes_runtime')) {
-                throw new Text_LanguageDetect_Exception(
-                    'Error loading database. Try turning magic_quotes_runtime off.',
-                    Text_LanguageDetect_Exception::MAGIC_QUOTES
-                );
-            }
-            throw new Text_LanguageDetect_Exception(
-                'Language database is not an array.',
-                Text_LanguageDetect_Exception::DB_NOT_ARRAY
-            );
-        } elseif (empty($trigram)) {
-            throw new Text_LanguageDetect_Exception(
-                'Language database has no elements.',
-                Text_LanguageDetect_Exception::DB_EMPTY
-            );
-        }
-    }
-
-    /**
-     * Omits languages
-     *
-     * Pass this function the name of or an array of names of
-     * languages that you don't want considered
-     *
-     * If you're only expecting a limited set of languages, this can greatly
-     * speed up processing
-     *
-     * @param mixed $omit_list    language name or array of names to omit
-     * @param bool  $include_only if true will include (rather than
-     *                            exclude) only those in the list
-     *
-     * @return int number of languages successfully deleted
-     * @throws Text_LanguageDetect_Exception
-     */
-    public function omitLanguages($omit_list, $include_only = false)
-    {
-        $deleted = 0;
-
-        $omit_list = $this->_convertFromNameMode($omit_list);
-
-        if (!$include_only) {
-            // deleting the given languages
-            if (!is_array($omit_list)) {
-                $omit_list = strtolower($omit_list); // case desensitize
-                if (isset($this->_lang_db[$omit_list])) {
-                    unset($this->_lang_db[$omit_list]);
-                    $deleted++;
-                }
-            } else {
-                foreach ($omit_list as $omit_lang) {
-                    if (isset($this->_lang_db[$omit_lang])) {
-                        unset($this->_lang_db[$omit_lang]);
-                        $deleted++;
-                    }
-                }
-            }
-
-        } else {
-            // deleting all except the given languages
-            if (!is_array($omit_list)) {
-                $omit_list = array($omit_list);
-            }
-
-            // case desensitize
-            foreach ($omit_list as $key => $omit_lang) {
-                $omit_list[$key] = strtolower($omit_lang);
-            }
-
-            foreach (array_keys($this->_lang_db) as $lang) {
-                if (!in_array($lang, $omit_list)) {
-                    unset($this->_lang_db[$lang]);
-                    $deleted++;
-                }
-            }
-        }
-
-        // reset the cluster cache if the number of languages changes
-        // this will then have to be recalculated
-        if (isset($this->_clusters) && $deleted > 0) {
-            $this->_clusters = null;
-        }
-
-        return $deleted;
-    }
-
-
-    /**
-     * Returns the number of languages that this object can detect
-     *
-     * @access public
-     * @return int            the number of languages
-     * @throws   Text_LanguageDetect_Exception
-     */
-    function getLanguageCount()
-    {
-        return count($this->_lang_db);
-    }
-
-    /**
-     * Checks if the language with the given name exists in the database
-     *
-     * @param mixed $lang Language name or array of language names
-     *
-     * @return bool true if language model exists
-     */
-    public function languageExists($lang)
-    {
-        $lang = $this->_convertFromNameMode($lang);
-
-        if (is_string($lang)) {
-            return isset($this->_lang_db[strtolower($lang)]);
-
-        } elseif (is_array($lang)) {
-            foreach ($lang as $test_lang) {
-                if (!isset($this->_lang_db[strtolower($test_lang)])) {
-                    return false;
-                }
-            }
-            return true;
-
-        } else {
-            throw new Text_LanguageDetect_Exception(
-                'Unsupported parameter type passed to languageExists()',
-                Text_LanguageDetect_Exception::PARAM_TYPE
-            );
-        }
-    }
-
-    /**
-     * Returns the list of detectable languages
-     *
-     * @access public
-     * @return array        the names of the languages known to this object<<<<<<<
-     * @throws   Text_LanguageDetect_Exception
-     */
-    function getLanguages()
-    {
-        return $this->_convertToNameMode(
-            array_keys($this->_lang_db)
-        );
-    }
-
-    /**
-     * Make this object behave like Language::Guess
-     *
-     * @param bool $setting false to turn off perl compatibility
-     *
-     * @return void
-     */
-    public function setPerlCompatible($setting = true)
-    {
-        if (is_bool($setting)) { // input check
-            $this->_perl_compatible = $setting;
-
-            if ($setting == true) {
-                $this->_max_score = $this->_threshold;
-            } else {
-                $this->_max_score = 0;
-            }
-        }
-
-    }
-
-    /**
-     * Sets the way how language names are accepted and returned.
-     *
-     * @param integer $name_mode One of the following modes:
-     *                           0 - language name ("english")
-     *                           2 - 2-letter ISO 639-1 code ("en")
-     *                           3 - 3-letter ISO 639-2 code ("eng")
-     *
-     * @return void
-     */
-    function setNameMode($name_mode)
-    {
-        $this->_name_mode = $name_mode;
-    }
-
-    /**
-     * Whether to use unicode block ranges in detection
-     *
-     * Should speed up most detections if turned on (detault is on). In some
-     * circumstances it may be slower, such as for large text samples (> 10K)
-     * in languages that use latin scripts. In other cases it should speed up
-     * detection noticeably.
-     *
-     * @param bool $setting false to turn off
-     *
-     * @return void
-     */
-    public function useUnicodeBlocks($setting = true)
-    {
-        if (is_bool($setting)) {
-            $this->_use_unicode_narrowing = $setting;
-        }
-    }
-
-    /**
-     * Converts a piece of text into trigrams
-     *
-     * @param string $text text to convert
-     *
-     * @return     array array of trigram frequencies
-     * @access     private
-     * @deprecated Superceded by the Text_LanguageDetect_Parser class
-     */
-    function _trigram($text)
-    {
-        $s = new Text_LanguageDetect_Parser($text);
-        $s->prepareTrigram();
-        $s->prepareUnicode(false);
-        $s->setPadStart(!$this->_perl_compatible);
-        $s->analyze();
-        return $s->getTrigramFreqs();
-    }
-
-    /**
-     * Converts a set of trigrams from frequencies to ranks
-     *
-     * Thresholds (cuts off) the list at $this->_threshold
-     *
-     * @param array $arr array of trigram
-     *
-     * @return array ranks of trigrams
-     * @access protected
-     */
-    function _arr_rank($arr)
-    {
-
-        // sorts alphabetically first as a standard way of breaking rank ties
-        $this->_bub_sort($arr);
-
-        // below might also work, but seemed to introduce errors in testing
-        //ksort($arr);
-        //asort($arr);
-
-        $rank = array();
-
-        $i = 0;
-        foreach ($arr as $key => $value) {
-            $rank[$key] = $i++;
-
-            // cut off at a standard threshold
-            if ($i >= $this->_threshold) {
-                break;
-            }
-        }
-
-        return $rank;
-    }
-
-    /**
-     * Sorts an array by value breaking ties alphabetically
-     *
-     * @param array &$arr the array to sort
-     *
-     * @return void
-     * @access private
-     */
-    function _bub_sort(&$arr)
-    {
-        // should do the same as this perl statement:
-        // sort { $trigrams{$b} == $trigrams{$a}
-        //   ?  $a cmp $b : $trigrams{$b} <=> $trigrams{$a} }
-
-        // needs to sort by both key and value at once
-        // using the key to break ties for the value
-
-        // converts array into an array of arrays of each key and value
-        // may be a better way of doing this
-        $combined = array();
-
-        foreach ($arr as $key => $value) {
-            $combined[] = array($key, $value);
-        }
-
-        usort($combined, array($this, '_sort_func'));
-
-        $replacement = array();
-        foreach ($combined as $key => $value) {
-            list($new_key, $new_value) = $value;
-            $replacement[$new_key] = $new_value;
-        }
-
-        $arr = $replacement;
-    }
-
-    /**
-     * Sort function used by bubble sort
-     *
-     * Callback function for usort().
-     *
-     * @param array $a first param passed by usort()
-     * @param array $b second param passed by usort()
-     *
-     * @return int 1 if $a is greater, -1 if not
-     * @see    _bub_sort()
-     * @access private
-     */
-    function _sort_func($a, $b)
-    {
-        // each is actually a key/value pair, so that it can compare using both
-        list($a_key, $a_value) = $a;
-        list($b_key, $b_value) = $b;
-
-        if ($a_value == $b_value) {
-            // if the values are the same, break ties using the key
-            return strcmp($a_key, $b_key);
-
-        } else {
-            // if not, just sort normally
-            if ($a_value > $b_value) {
-                return -1;
-            } else {
-                return 1;
-            }
-        }
-
-        // 0 should not be possible because keys must be unique
-    }
-
-    /**
-     * Calculates a linear rank-order distance statistic between two sets of
-     * ranked trigrams
-     *
-     * Sums the differences in rank for each trigram. If the trigram does not
-     * appear in both, consider it a difference of $this->_threshold.
-     *
-     * This distance measure was proposed by Cavnar & Trenkle (1994). Despite
-     * its simplicity it has been shown to be highly accurate for language
-     * identification tasks.
-     *
-     * @param array $arr1 the reference set of trigram ranks
-     * @param array $arr2 the target set of trigram ranks
-     *
-     * @return int the sum of the differences between the ranks of
-     *             the two trigram sets
-     * @access private
-     */
-    function _distance($arr1, $arr2)
-    {
-        $sumdist = 0;
-
-        foreach ($arr2 as $key => $value) {
-            if (isset($arr1[$key])) {
-                $distance = abs($value - $arr1[$key]);
-            } else {
-                // $this->_threshold sets the maximum possible distance value
-                // for any one pair of trigrams
-                $distance = $this->_threshold;
-            }
-            $sumdist += $distance;
-        }
-
-        return $sumdist;
-
-        // todo: there are other distance statistics to try, e.g. relative
-        //       entropy, but they're probably more costly to compute
-    }
-
-    /**
-     * Normalizes the score returned by _distance()
-     *
-     * Different if perl compatible or not
-     *
-     * @param int $score      the score from _distance()
-     * @param int $base_count the number of trigrams being considered
-     *
-     * @return float the normalized score
-     * @see    _distance()
-     * @access private
-     */
-    function _normalize_score($score, $base_count = null)
-    {
-        if ($base_count === null) {
-            $base_count = $this->_threshold;
-        }
-
-        if (!$this->_perl_compatible) {
-            return 1 - ($score / $base_count / $this->_threshold);
-        } else {
-            return floor($score / $base_count);
-        }
-    }
-
-
-    /**
-     * Detects the closeness of a sample of text to the known languages
-     *
-     * Calculates the statistical difference between the text and
-     * the trigrams for each language, normalizes the score then
-     * returns results for all languages in sorted order
-     *
-     * If perl compatible, the score is 300-0, 0 being most similar.
-     * Otherwise, it's 0-1 with 1 being most similar.
-     *
-     * The $sample text should be at least a few sentences in length;
-     * should be ascii-7 or utf8 encoded, if another and the mbstring extension
-     * is present it will try to detect and convert. However, experience has
-     * shown that mb_detect_encoding() *does not work very well* with at least
-     * some types of encoding.
-     *
-     * @param string $sample a sample of text to compare.
-     * @param int    $limit  if specified, return an array of the most likely
-     *                       $limit languages and their scores.
-     *
-     * @return mixed sorted array of language scores, blank array if no
-     *               useable text was found
-     * @see    _distance()
-     * @throws Text_LanguageDetect_Exception
-     */
-    public function detect($sample, $limit = 0)
-    {
-        // input check
-        if (!Text_LanguageDetect_Parser::validateString($sample)) {
-            return array();
-        }
-
-        // check char encoding
-        // (only if mbstring extension is compiled and PHP > 4.0.6)
-        if (function_exists('mb_detect_encoding')
-            && function_exists('mb_convert_encoding')
-        ) {
-            // mb_detect_encoding isn't very reliable, to say the least
-            // detection should still work with a sufficient sample
-            //  of ascii characters
-            $encoding = mb_detect_encoding($sample);
-
-            // mb_detect_encoding() will return FALSE if detection fails
-            // don't attempt conversion if that's the case
-            if ($encoding != 'ASCII' && $encoding != 'UTF-8'
-                && $encoding !== false
-            ) {
-                // verify the encoding exists in mb_list_encodings
-                if (in_array($encoding, mb_list_encodings())) {
-                    $sample = mb_convert_encoding($sample, 'UTF-8', $encoding);
-                }
-            }
-        }
-
-        $sample_obj = new Text_LanguageDetect_Parser($sample);
-        $sample_obj->prepareTrigram();
-        if ($this->_use_unicode_narrowing) {
-            $sample_obj->prepareUnicode();
-        }
-        $sample_obj->setPadStart(!$this->_perl_compatible);
-        $sample_obj->analyze();
-
-        $trigram_freqs =& $sample_obj->getTrigramRanks();
-        $trigram_count = count($trigram_freqs);
-
-        if ($trigram_count == 0) {
-            return array();
-        }
-
-        $scores = array();
-
-        // use unicode block detection to narrow down the possibilities
-        if ($this->_use_unicode_narrowing) {
-            $blocks =& $sample_obj->getUnicodeBlocks();
-
-            if (is_array($blocks)) {
-                $present_blocks = array_keys($blocks);
-            } else {
-                throw new Text_LanguageDetect_Exception(
-                    'Error during block detection',
-                    Text_LanguageDetect_Exception::BLOCK_DETECTION
-                );
-            }
-
-            $possible_langs = array();
-
-            foreach ($present_blocks as $blockname) {
-                if (isset($this->_unicode_map[$blockname])) {
-
-                    $possible_langs = array_merge(
-                        $possible_langs,
-                        array_keys($this->_unicode_map[$blockname])
-                    );
-
-                    // todo: faster way to do this?
-                }
-            }
-
-            // could also try an intersect operation rather than a union
-            // in other words, choose languages whose trigrams contain
-            // ALL of the unicode blocks found in this sample
-            // would improve speed but would be completely thrown off by an
-            // unexpected character, like an umlaut appearing in english text
-
-            $possible_langs = array_intersect(
-                array_keys($this->_lang_db),
-                array_unique($possible_langs)
-            );
-
-            // needs to intersect it with the keys of _lang_db in case
-            // languages have been omitted
-
-        } else {
-            // or just try 'em all
-            $possible_langs = array_keys($this->_lang_db);
-        }
-
-
-        foreach ($possible_langs as $lang) {
-            $scores[$lang] = $this->_normalize_score(
-                $this->_distance($this->_lang_db[$lang], $trigram_freqs),
-                $trigram_count
-            );
-        }
-
-        unset($sample_obj);
-
-        if ($this->_perl_compatible) {
-            asort($scores);
-        } else {
-            arsort($scores);
-        }
-
-        // todo: drop languages with a score of $this->_max_score?
-
-        // limit the number of returned scores
-        if ($limit && is_numeric($limit)) {
-            $limited_scores = array();
-
-            $i = 0;
-            foreach ($scores as $key => $value) {
-                if ($i++ >= $limit) {
-                    break;
-                }
-
-                $limited_scores[$key] = $value;
-            }
-
-            return $this->_convertToNameMode($limited_scores, true);
-        } else {
-            return $this->_convertToNameMode($scores, true);
-        }
-    }
-
-    /**
-     * Returns only the most similar language to the text sample
-     *
-     * Calls $this->detect() and returns only the top result
-     *
-     * @param string $sample text to detect the language of
-     *
-     * @return string the name of the most likely language
-     *                or null if no language is similar
-     * @see    detect()
-     * @throws Text_LanguageDetect_Exception
-     */
-    public function detectSimple($sample)
-    {
-        $scores = $this->detect($sample, 1);
-
-        // if top language has the maximum possible score,
-        // then the top score will have been picked at random
-        if (!is_array($scores) || empty($scores)
-            || current($scores) == $this->_max_score
-        ) {
-            return null;
-        } else {
-            return key($scores);
-        }
-    }
-
-    /**
-     * Returns an array containing the most similar language and a confidence
-     * rating
-     *
-     * Confidence is a simple measure calculated from the similarity score
-     * minus the similarity score from the next most similar language
-     * divided by the highest possible score. Languages that have closely
-     * related cousins (e.g. Norwegian and Danish) should generally have lower
-     * confidence scores.
-     *
-     * The similarity score answers the question "How likely is the text the
-     * returned language regardless of the other languages considered?" The
-     * confidence score is one way of answering the question "how likely is the
-     * text the detected language relative to the rest of the language model
-     * set?"
-     *
-     * To see how similar languages are a priori, see languageSimilarity()
-     *
-     * @param string $sample text for which language will be detected
-     *
-     * @return array most similar language, score and confidence rating
-     *               or null if no language is similar
-     * @see    detect()
-     * @throws Text_LanguageDetect_Exception
-     */
-    public function detectConfidence($sample)
-    {
-        $scores = $this->detect($sample, 2);
-
-        // if most similar language has the max score, it
-        // will have been picked at random
-        if (!is_array($scores) || empty($scores)
-            || current($scores) == $this->_max_score
-        ) {
-            return null;
-        }
-
-        $arr['language'] = key($scores);
-        $arr['similarity'] = current($scores);
-        if (next($scores) !== false) { // if false then no next element
-            // the goal is to return a higher value if the distance between
-            // the similarity of the first score and the second score is high
-
-            if ($this->_perl_compatible) {
-                $arr['confidence'] = (current($scores) - $arr['similarity'])
-                    / $this->_max_score;
-
-            } else {
-                $arr['confidence'] = $arr['similarity'] - current($scores);
-
-            }
-
-        } else {
-            $arr['confidence'] = null;
-        }
-
-        return $arr;
-    }
-
-    /**
-     * Returns the distribution of unicode blocks in a given utf8 string
-     *
-     * For the block name of a single char, use unicodeBlockName()
-     *
-     * @param string $str          input string. Must be ascii or utf8
-     * @param bool   $skip_symbols if true, skip ascii digits, symbols and
-     *                             non-printing characters. Includes spaces,
-     *                             newlines and common punctutation characters.
-     *
-     * @return array
-     * @throws Text_LanguageDetect_Exception
-     */
-    public function detectUnicodeBlocks($str, $skip_symbols)
-    {
-        $skip_symbols = (bool)$skip_symbols;
-        $str          = (string)$str;
-
-        $sample_obj = new Text_LanguageDetect_Parser($str);
-        $sample_obj->prepareUnicode();
-        $sample_obj->prepareTrigram(false);
-        $sample_obj->setUnicodeSkipSymbols($skip_symbols);
-        $sample_obj->analyze();
-        $blocks = $sample_obj->getUnicodeBlocks();
-        unset($sample_obj);
-        return $blocks;
-    }
-
-    /**
-     * Returns the block name for a given unicode value
-     *
-     * If passed a string, will assume it is being passed a UTF8-formatted
-     * character and will automatically convert. Otherwise it will assume it
-     * is being passed a numeric unicode value.
-     *
-     * Make sure input is of the correct type!
-     *
-     * @param mixed $unicode unicode value or utf8 char
-     *
-     * @return mixed the block name string or false if not found
-     * @throws Text_LanguageDetect_Exception
-     */
-    public function unicodeBlockName($unicode)
-    {
-        if (is_string($unicode)) {
-            // assume it is being passed a utf8 char, so convert it
-            if (self::utf8strlen($unicode) > 1) {
-                throw new Text_LanguageDetect_Exception(
-                    'Pass a single char only to this method',
-                    Text_LanguageDetect_Exception::PARAM_TYPE
-                );
-            }
-            $unicode = $this->_utf8char2unicode($unicode);
-
-        } elseif (!is_int($unicode)) {
-            throw new Text_LanguageDetect_Exception(
-                'Input must be of type string or int.',
-                Text_LanguageDetect_Exception::PARAM_TYPE
-            );
-        }
-
-        $blocks = $this->_read_unicode_block_db();
-
-        $result = $this->_unicode_block_name($unicode, $blocks);
-
-        if ($result == -1) {
-            return false;
-        } else {
-            return $result[2];
-        }
-    }
-
-    /**
-     * Searches the unicode block database
-     *
-     * Returns the block name for a given unicode value. unicodeBlockName() is
-     * the public interface for this function, which does input checks which
-     * this function omits for speed.
-     *
-     * @param int   $unicode     the unicode value
-     * @param array $blocks      the block database
-     * @param int   $block_count the number of defined blocks in the database
-     *
-     * @return mixed Block name, -1 if it failed
-     * @see    unicodeBlockName()
-     * @access protected
-     */
-    function _unicode_block_name($unicode, $blocks, $block_count = -1)
-    {
-        // for a reference, see
-        // http://www.unicode.org/Public/UNIDATA/Blocks.txt
-
-        // assume that ascii characters are the most common
-        // so try it first for efficiency
-        if ($unicode <= $blocks[0][1]) {
-            return $blocks[0];
-        }
-
-        // the optional $block_count param is for efficiency
-        // so we this function doesn't have to run count() every time
-        if ($block_count != -1) {
-            $high = $block_count - 1;
-        } else {
-            $high = count($blocks) - 1;
-        }
-
-        $low = 1; // start with 1 because ascii was 0
-
-        // your average binary search algorithm
-        while ($low <= $high) {
-            $mid = floor(($low + $high) / 2);
-
-            if ($unicode < $blocks[$mid][0]) {
-                // if it's lower than the lower bound
-                $high = $mid - 1;
-
-            } elseif ($unicode > $blocks[$mid][1]) {
-                // if it's higher than the upper bound
-                $low = $mid + 1;
-
-            } else {
-                // found it
-                return $blocks[$mid];
-            }
-        }
-
-        // failed to find the block
-        return -1;
-
-        // todo: differentiate when it's out of range or when it falls
-        //       into an unassigned range?
-    }
-
-    /**
-     * Brings up the unicode block database
-     *
-     * @return array the database of unicode block definitions
-     * @throws Text_LanguageDetect_Exception
-     * @access protected
-     */
-    function _read_unicode_block_db()
-    {
-        // since the unicode definitions are always going to be the same,
-        // might as well share the memory for the db with all other instances
-        // of this class
-        static $data;
-
-        if (!isset($data)) {
-            $data = $this->_readdb($this->_unicode_db_filename);
-        }
-
-        return $data;
-    }
-
-    /**
-     * Calculate the similarities between the language models
-     *
-     * Use this function to see how similar languages are to each other.
-     *
-     * If passed 2 language names, will return just those languages compared.
-     * If passed 1 language name, will return that language compared to
-     * all others.
-     * If passed none, will return an array of every language model compared
-     * to every other one.
-     *
-     * @param string $lang1 the name of the first language to be compared
-     * @param string $lang2 the name of the second language to be compared
-     *
-     * @return array scores of every language compared
-     *               or the score of just the provided languages
-     *               or null if one of the supplied languages does not exist
-     * @throws Text_LanguageDetect_Exception
-     */
-    public function languageSimilarity($lang1 = null, $lang2 = null)
-    {
-        $lang1 = $this->_convertFromNameMode($lang1);
-        $lang2 = $this->_convertFromNameMode($lang2);
-        if ($lang1 != null) {
-            $lang1 = strtolower($lang1);
-
-            // check if language model exists
-            if (!isset($this->_lang_db[$lang1])) {
-                return null;
-            }
-
-            if ($lang2 != null) {
-                if (!isset($this->_lang_db[$lang2])) {
-                    // check if language model exists
-                    return null;
-                }
-
-                $lang2 = strtolower($lang2);
-
-                // compare just these two languages
-                return $this->_normalize_score(
-                    $this->_distance(
-                        $this->_lang_db[$lang1],
-                        $this->_lang_db[$lang2]
-                    )
-                );
-
-            } else {
-                // compare just $lang1 to all languages
-                $return_arr = array();
-                foreach ($this->_lang_db as $key => $value) {
-                    if ($key != $lang1) {
-                        // don't compare a language to itself
-                        $return_arr[$key] = $this->_normalize_score(
-                            $this->_distance($this->_lang_db[$lang1], $value)
-                        );
-                    }
-                }
-                asort($return_arr);
-
-                return $return_arr;
-            }
-
-
-        } else {
-            // compare all languages to each other
-            $return_arr = array();
-            foreach (array_keys($this->_lang_db) as $lang1) {
-                foreach (array_keys($this->_lang_db) as $lang2) {
-                    // skip comparing languages to themselves
-                    if ($lang1 != $lang2) {
-
-                        if (isset($return_arr[$lang2][$lang1])) {
-                            // don't re-calculate what's already been done
-                            $return_arr[$lang1][$lang2]
-                                = $return_arr[$lang2][$lang1];
-
-                        } else {
-                            // calculate
-                            $return_arr[$lang1][$lang2]
-                                = $this->_normalize_score(
-                                    $this->_distance(
-                                        $this->_lang_db[$lang1],
-                                        $this->_lang_db[$lang2]
-                                    )
-                                );
-
-                        }
-                    }
-                }
-            }
-            return $return_arr;
-        }
-    }
-
-    /**
-     * Cluster known languages according to languageSimilarity()
-     *
-     * WARNING: this method is EXPERIMENTAL. It is not recommended for common
-     * use, and it may disappear or its functionality may change in future
-     * releases without notice.
-     *
-     * Uses a nearest neighbor technique to generate the maximum possible
-     * number of dendograms from the similarity data.
-     *
-     * @access      public
-     * @return      array language cluster data
-     * @throws      Text_LanguageDetect_Exception
-     * @see         languageSimilarity()
-     * @deprecated  this function will eventually be removed and placed into
-     *              the model generation class
-     */
-    function clusterLanguages()
-    {
-        // todo: set the maximum number of clusters
-        // return cached result, if any
-        if (isset($this->_clusters)) {
-            return $this->_clusters;
-        }
-
-        $langs = array_keys($this->_lang_db);
-
-        $arr = $this->languageSimilarity();
-
-        sort($langs);
-
-        foreach ($langs as $lang) {
-            if (!isset($this->_lang_db[$lang])) {
-                throw new Text_LanguageDetect_Exception(
-                    "missing $lang!",
-                    Text_LanguageDetect_Exception::UNKNOWN_LANGUAGE
-                );
-            }
-        }
-
-        // http://www.psychstat.missouristate.edu/multibook/mlt04m.html
-        foreach ($langs as $old_key => $lang1) {
-            $langs[$lang1] = $lang1;
-            unset($langs[$old_key]);
-        }
-
-        $result_data = $really_map = array();
-
-        $i = 0;
-        while (count($langs) > 2 && $i++ < 200) {
-            $highest_score = -1;
-            $highest_key1 = '';
-            $highest_key2 = '';
-            foreach ($langs as $lang1) {
-                foreach ($langs as $lang2) {
-                    if ($lang1 != $lang2
-                        && $arr[$lang1][$lang2] > $highest_score
-                    ) {
-                        $highest_score = $arr[$lang1][$lang2];
-                        $highest_key1 = $lang1;
-                        $highest_key2 = $lang2;
-                    }
-                }
-            }
-
-            if (!$highest_key1) {
-                // should not ever happen
-                throw new Text_LanguageDetect_Exception(
-                    "no highest key? (step: $i)",
-                    Text_LanguageDetect_Exception::NO_HIGHEST_KEY
-                );
-            }
-
-            if ($highest_score == 0) {
-                // languages are perfectly dissimilar
-                break;
-            }
-
-            // $highest_key1 and $highest_key2 are most similar
-            $sum1 = array_sum($arr[$highest_key1]);
-            $sum2 = array_sum($arr[$highest_key2]);
-
-            // use the score for the one that is most similar to the rest of
-            // the field as the score for the group
-            // todo: could try averaging or "centroid" method instead
-            // seems like that might make more sense
-            // actually nearest neighbor may be better for binary searching
-
-
-            // for "Complete Linkage"/"furthest neighbor"
-            // sign should be <
-            // for "Single Linkage"/"nearest neighbor" method
-            // should should be >
-            // results seem to be pretty much the same with either method
-
-            // figure out which to delete and which to replace
-            if ($sum1 > $sum2) {
-                $replaceme = $highest_key1;
-                $deleteme = $highest_key2;
-            } else {
-                $replaceme = $highest_key2;
-                $deleteme = $highest_key1;
-            }
-
-            $newkey = $replaceme . ':' . $deleteme;
-
-            // $replaceme is most similar to remaining languages
-            // replace $replaceme with '$newkey', deleting $deleteme
-
-            // keep a record of which fork is really which language
-            $really_lang = $replaceme;
-            while (isset($really_map[$really_lang])) {
-                $really_lang = $really_map[$really_lang];
-            }
-            $really_map[$newkey] = $really_lang;
-
-
-            // replace the best fitting key, delete the other
-            foreach ($arr as $key1 => $arr2) {
-                foreach ($arr2 as $key2 => $value2) {
-                    if ($key2 == $replaceme) {
-                        $arr[$key1][$newkey] = $arr[$key1][$key2];
-                        unset($arr[$key1][$key2]);
-                        // replacing $arr[$key1][$key2] with $arr[$key1][$newkey]
-                    }
-
-                    if ($key1 == $replaceme) {
-                        $arr[$newkey][$key2] = $arr[$key1][$key2];
-                        unset($arr[$key1][$key2]);
-                        // replacing $arr[$key1][$key2] with $arr[$newkey][$key2]
-                    }
-
-                    if ($key1 == $deleteme || $key2 == $deleteme) {
-                        // deleting $arr[$key1][$key2]
-                        unset($arr[$key1][$key2]);
-                    }
-                }
-            }
-
-
-            unset($langs[$highest_key1]);
-            unset($langs[$highest_key2]);
-            $langs[$newkey] = $newkey;
-
-
-            // some of these may be overkill
-            $result_data[$newkey] = array(
-                                'newkey' => $newkey,
-                                'count' => $i,
-                                'diff' => abs($sum1 - $sum2),
-                                'score' => $highest_score,
-                                'bestfit' => $replaceme,
-                                'otherfit' => $deleteme,
-                                'really' => $really_lang,
-                            );
-        }
-
-        $return_val = array(
-                'open_forks' => $langs,
-                        // the top level of clusters
-                        // clusters that are mutually exclusive
-                        // or specified by a specific maximum
-
-                'fork_data' => $result_data,
-                        // data for each split
-
-                'name_map' => $really_map,
-                        // which cluster is really which language
-                        // using the nearest neighbor technique, the cluster
-                        // inherits all of the properties of its most-similar member
-                        // this keeps track
-            );
-
-
-        // saves the result in the object
-        $this->_clusters = $return_val;
-
-        return $return_val;
-    }
-
-
-    /**
-     * Perform an intelligent detection based on clusterLanguages()
-     *
-     * WARNING: this method is EXPERIMENTAL. It is not recommended for common
-     * use, and it may disappear or its functionality may change in future
-     * releases without notice.
-     *
-     * This compares the sample text to top the top level of clusters. If the
-     * sample is similar to the cluster it will drop down and compare it to the
-     * languages in the cluster, and so on until it hits a leaf node.
-     *
-     * this should find the language in considerably fewer compares
-     * (the equivalent of a binary search), however clusterLanguages() is costly
-     * and the loss of accuracy from this technique is significant.
-     *
-     * This method may need to be 'fuzzier' in order to become more accurate.
-     *
-     * This function could be more useful if the universe of possible languages
-     * was very large, however in such cases some method of Bayesian inference
-     * might be more helpful.
-     *
-     * @param string $str input string
-     *
-     * @return array language scores (only those compared)
-     * @throws Text_LanguageDetect_Exception
-     * @see    clusterLanguages()
-     */
-    public function clusteredSearch($str)
-    {
-        // input check
-        if (!Text_LanguageDetect_Parser::validateString($str)) {
-            return array();
-        }
-
-        // clusterLanguages() will return a cached result if possible
-        // so it's safe to call it every time
-        $result = $this->clusterLanguages();
-
-        $dendogram_start = $result['open_forks'];
-        $dendogram_data  = $result['fork_data'];
-        $dendogram_alias = $result['name_map'];
-
-        $sample_obj = new Text_LanguageDetect_Parser($str);
-        $sample_obj->prepareTrigram();
-        $sample_obj->setPadStart(!$this->_perl_compatible);
-        $sample_obj->analyze();
-        $sample_result = $sample_obj->getTrigramRanks();
-        $sample_count  = count($sample_result);
-
-        // input check
-        if ($sample_count == 0) {
-            return array();
-        }
-
-        $i = 0; // counts the number of steps
-
-        foreach ($dendogram_start as $lang) {
-            if (isset($dendogram_alias[$lang])) {
-                $lang_key = $dendogram_alias[$lang];
-            } else {
-                $lang_key = $lang;
-            }
-
-            $scores[$lang] = $this->_normalize_score(
-                $this->_distance($this->_lang_db[$lang_key], $sample_result),
-                $sample_count
-            );
-
-            $i++;
-        }
-
-        if ($this->_perl_compatible) {
-            asort($scores);
-        } else {
-            arsort($scores);
-        }
-
-        $top_score = current($scores);
-        $top_key = key($scores);
-
-        // of starting forks, $top_key is the most similar to the sample
-
-        $cur_key = $top_key;
-        while (isset($dendogram_data[$cur_key])) {
-            $lang1 = $dendogram_data[$cur_key]['bestfit'];
-            $lang2 = $dendogram_data[$cur_key]['otherfit'];
-            foreach (array($lang1, $lang2) as $lang) {
-                if (isset($dendogram_alias[$lang])) {
-                    $lang_key = $dendogram_alias[$lang];
-                } else {
-                    $lang_key = $lang;
-                }
-
-                $scores[$lang] = $this->_normalize_score(
-                    $this->_distance($this->_lang_db[$lang_key], $sample_result),
-                    $sample_count
-                );
-
-                //todo: does not need to do same comparison again
-            }
-
-            $i++;
-
-            if ($scores[$lang1] > $scores[$lang2]) {
-                $cur_key = $lang1;
-                $loser_key = $lang2;
-            } else {
-                $cur_key = $lang2;
-                $loser_key = $lang1;
-            }
-
-            $diff = $scores[$cur_key] - $scores[$loser_key];
-
-            // $cur_key ({$dendogram_alias[$cur_key]}) wins
-            // over $loser_key ({$dendogram_alias[$loser_key]})
-            // with a difference of $diff
-        }
-
-        // found result in $i compares
-
-        // rather than sorting the result, preserve it so that you can see
-        // which paths the algorithm decided to take along the tree
-
-        // but sometimes the last item is only the second highest
-        if (($this->_perl_compatible  && (end($scores) > prev($scores)))
-            || (!$this->_perl_compatible && (end($scores) < prev($scores)))
-        ) {
-            $real_last_score = current($scores);
-            $real_last_key = key($scores);
-
-            // swaps the 2nd-to-last item for the last item
-            unset($scores[$real_last_key]);
-            $scores[$real_last_key] = $real_last_score;
-        }
-
-
-        if (!$this->_perl_compatible) {
-            $scores = array_reverse($scores, true);
-            // second param requires php > 4.0.3
-        }
-
-        return $scores;
-    }
-
-    /**
-     * ut8-safe strlen()
-     *
-     * Returns the numbers of characters (not bytes) in a utf8 string
-     *
-     * @param string $str string to get the length of
-     *
-     * @return int number of chars
-     */
-    public static function utf8strlen($str)
-    {
-        // utf8_decode() will convert unknown chars to '?', which is actually
-        // ideal for counting.
-
-        return strlen(utf8_decode($str));
-
-        // idea stolen from dokuwiki
-    }
-
-    /**
-     * Returns the unicode value of a utf8 char
-     *
-     * @param string $char a utf8 (possibly multi-byte) char
-     *
-     * @return int unicode value
-     * @access protected
-     * @link   http://en.wikipedia.org/wiki/UTF-8
-     */
-    function _utf8char2unicode($char)
-    {
-        // strlen() here will actually get the binary length of a single char
-        switch (strlen($char)) {
-        case 1:
-            // normal ASCII-7 byte
-            // 0xxxxxxx -->  0xxxxxxx
-            return ord($char{0});
-
-        case 2:
-            // 2 byte unicode
-            // 110zzzzx 10xxxxxx --> 00000zzz zxxxxxxx
-            $z = (ord($char{0}) & 0x000001F) << 6;
-            $x = (ord($char{1}) & 0x0000003F);
-            return ($z | $x);
-
-        case 3:
-            // 3 byte unicode
-            // 1110zzzz 10zxxxxx 10xxxxxx --> zzzzzxxx xxxxxxxx
-            $z =  (ord($char{0}) & 0x0000000F) << 12;
-            $x1 = (ord($char{1}) & 0x0000003F) << 6;
-            $x2 = (ord($char{2}) & 0x0000003F);
-            return ($z | $x1 | $x2);
-
-        case 4:
-            // 4 byte unicode
-            // 11110zzz 10zzxxxx 10xxxxxx 10xxxxxx -->
-            // 000zzzzz xxxxxxxx xxxxxxxx
-            $z1 = (ord($char{0}) & 0x00000007) << 18;
-            $z2 = (ord($char{1}) & 0x0000003F) << 12;
-            $x1 = (ord($char{2}) & 0x0000003F) << 6;
-            $x2 = (ord($char{3}) & 0x0000003F);
-            return ($z1 | $z2 | $x1 | $x2);
-        }
-    }
-
-    /**
-     * utf8-safe fast character iterator
-     *
-     * Will get the next character starting from $counter, which will then be
-     * incremented. If a multi-byte char the bytes will be concatenated and
-     * $counter will be incremeted by the number of bytes in the char.
-     *
-     * @param string $str             the string being iterated over
-     * @param int    &$counter        the iterator, will increment by reference
-     * @param bool   $special_convert whether to do special conversions
-     *
-     * @return char the next (possibly multi-byte) char from $counter
-     * @access private
-     */
-    static function _next_char($str, &$counter, $special_convert = false)
-    {
-        $char = $str{$counter++};
-        $ord = ord($char);
-
-        // for a description of the utf8 system see
-        // http://www.phpclasses.org/browse/file/5131.html
-
-        // normal ascii one byte char
-        if ($ord <= 127) {
-            // special conversions needed for this package
-            // (that only apply to regular ascii characters)
-            // lower case, and convert all non-alphanumeric characters
-            // other than "'" to space
-            if ($special_convert && $char != ' ' && $char != "'") {
-                if ($ord >= 65 && $ord <= 90) { // A-Z
-                    $char = chr($ord + 32); // lower case
-                } elseif ($ord < 97 || $ord > 122) { // NOT a-z
-                    $char = ' '; // convert to space
-                }
-            }
-
-            return $char;
-
-        } elseif ($ord >> 5 == 6) { // two-byte char
-            // multi-byte chars
-            $nextchar = $str{$counter++}; // get next byte
-
-            // lower-casing of non-ascii characters is still incomplete
-
-            if ($special_convert) {
-                // lower case latin accented characters
-                if ($ord == 195) {
-                    $nextord = ord($nextchar);
-                    $nextord_adj = $nextord + 64;
-                    // for a reference, see
-                    // http://www.ramsch.org/martin/uni/fmi-hp/iso8859-1.html
-
-                    // &Agrave; - &THORN; but not &times;
-                    if ($nextord_adj >= 192
-                        && $nextord_adj <= 222
-                        && $nextord_adj != 215
-                    ) {
-                        $nextchar = chr($nextord + 32);
-                    }
-
-                } elseif ($ord == 208) {
-                    // lower case cyrillic alphabet
-                    $nextord = ord($nextchar);
-                    // if A - Pe
-                    if ($nextord >= 144 && $nextord <= 159) {
-                        // lower case
-                        $nextchar = chr($nextord + 32);
-
-                    } elseif ($nextord >= 160 && $nextord <= 175) {
-                        // if Er - Ya
-                        // lower case
-                        $char = chr(209); // == $ord++
-                        $nextchar = chr($nextord - 32);
-                    }
-                }
-            }
-
-            // tag on next byte
-            return $char . $nextchar;
-        } elseif ($ord >> 4  == 14) { // three-byte char
-
-            // tag on next 2 bytes
-            return $char . $str{$counter++} . $str{$counter++};
-
-        } elseif ($ord >> 3 == 30) { // four-byte char
-
-            // tag on next 3 bytes
-            return $char . $str{$counter++} . $str{$counter++} . $str{$counter++};
-
-        } else {
-            // error?
-        }
-    }
-
-    /**
-     * Converts an $language input parameter from the configured mode
-     * to the language name that is used internally.
-     *
-     * Works for strings and arrays.
-     *
-     * @param string|array $lang       A language description ("english"/"en"/"eng")
-     * @param boolean      $convertKey If $lang is an array, setting $key
-     *                                 converts the keys to the language name.
-     *
-     * @return string|array Language name
-     */
-    function _convertFromNameMode($lang, $convertKey = false)
-    {
-        if ($this->_name_mode == 0) {
-            return $lang;
-        }
-
-        if ($this->_name_mode == 2) {
-            $method = 'code2ToName';
-        } else {
-            $method = 'code3ToName';
-        }
-
-        if (is_string($lang)) {
-            return (string)Text_LanguageDetect_ISO639::$method($lang);
-        }
-
-        $newlang = array();
-        foreach ($lang as $key => $val) {
-            if ($convertKey) {
-                $newkey = (string)Text_LanguageDetect_ISO639::$method($key);
-                $newlang[$newkey] = $val;
-            } else {
-                $newlang[$key] = (string)Text_LanguageDetect_ISO639::$method($val);
-            }
-        }
-        return $newlang;
-    }
-
-    /**
-     * Converts an $language output parameter from the language name that is
-     * used internally to the configured mode.
-     *
-     * Works for strings and arrays.
-     *
-     * @param string|array $lang       A language description ("english"/"en"/"eng")
-     * @param boolean      $convertKey If $lang is an array, setting $key
-     *                                 converts the keys to the language name.
-     *
-     * @return string|array Language name
-     */
-    function _convertToNameMode($lang, $convertKey = false)
-    {
-        if ($this->_name_mode == 0) {
-            return $lang;
-        }
-
-        if ($this->_name_mode == 2) {
-            $method = 'nameToCode2';
-        } else {
-            $method = 'nameToCode3';
-        }
-
-        if (is_string($lang)) {
-            return Text_LanguageDetect_ISO639::$method($lang);
-        }
-
-        $newlang = array();
-        foreach ($lang as $key => $val) {
-            if ($convertKey) {
-                $newkey = Text_LanguageDetect_ISO639::$method($key);
-                $newlang[$newkey] = $val;
-            } else {
-                $newlang[$key] = Text_LanguageDetect_ISO639::$method($val);
-            }
-        }
-        return $newlang;
-    }
-}
-
-/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
-
-?>
diff --git a/library/langdet/Text/LanguageDetect/Exception.php b/library/langdet/Text/LanguageDetect/Exception.php
deleted file mode 100644
index 196d994f5..000000000
--- a/library/langdet/Text/LanguageDetect/Exception.php
+++ /dev/null
@@ -1,57 +0,0 @@
-<?php
-class Text_LanguageDetect_Exception extends Exception
-{
-    /**
-     * Database file could not be found
-     */
-    const DB_NOT_FOUND = 10;
-
-    /**
-     * Database file found, but not readable
-     */
-    const DB_NOT_READABLE = 11;
-
-    /**
-     * Database file is empty
-     */
-    const DB_EMPTY = 12;
-
-    /**
-     * Database contents is not a PHP array
-     */
-    const DB_NOT_ARRAY = 13;
-
-    /**
-     * Magic quotes are activated
-     */
-    const MAGIC_QUOTES = 14;
-
-
-    /**
-     * Parameter of invalid type passed to method
-     */
-    const PARAM_TYPE = 20;
-
-    /**
-     * Character in parameter is invalid
-     */
-    const INVALID_CHAR = 21;
-
-
-    /**
-     * Language is not in the database
-     */
-    const UNKNOWN_LANGUAGE = 30;
-
-
-    /**
-     * Error during block detection
-     */
-    const BLOCK_DETECTION = 40;
-
-
-    /**
-     * Error while clustering languages
-     */
-    const NO_HIGHEST_KEY = 50;
-}
diff --git a/library/langdet/Text/LanguageDetect/ISO639.php b/library/langdet/Text/LanguageDetect/ISO639.php
deleted file mode 100644
index c577a2e1a..000000000
--- a/library/langdet/Text/LanguageDetect/ISO639.php
+++ /dev/null
@@ -1,341 +0,0 @@
-<?php
-/**
- * Part of Text_LanguageDetect
- *
- * PHP version 5
- *
- * @category  Text
- * @package   Text_LanguageDetect
- * @author    Christian Weiske <cweiske@php.net>
- * @copyright 2011 Christian Weiske <cweiske@php.net>
- * @license   http://www.debian.org/misc/bsd.license BSD
- * @version   SVN: $Id$
- * @link      http://pear.php.net/package/Text_LanguageDetect/
- */
-
-/**
- * Provides a mapping between the languages from lang.dat and the
- * ISO 639-1 and ISO-639-2 codes.
- *
- * Note that this class contains only languages that exist in lang.dat.
- *
- * @category  Text
- * @package   Text_LanguageDetect
- * @author    Christian Weiske <cweiske@php.net>
- * @copyright 2011 Christian Weiske <cweiske@php.net>
- * @license   http://www.debian.org/misc/bsd.license BSD
- * @link      http://www.loc.gov/standards/iso639-2/php/code_list.php
- */
-class Text_LanguageDetect_ISO639
-{
-    /**
-     * Maps all language names from the language database to the
-     * ISO 639-1 2-letter language code.
-     *
-     * NULL indicates that there is no 2-letter code.
-     *
-     * @var array
-     */
-    public static $nameToCode2 = array(
-        'albanian'   => 'sq',
-        'arabic'     => 'ar',
-        'azeri'      => 'az',
-        'bengali'    => 'bn',
-        'bulgarian'  => 'bg',
-        'cebuano'    => null,
-        'croatian'   => 'hr',
-        'czech'      => 'cs',
-        'danish'     => 'da',
-        'dutch'      => 'nl',
-        'english'    => 'en',
-        'estonian'   => 'et',
-        'farsi'      => 'fa',
-        'finnish'    => 'fi',
-        'french'     => 'fr',
-        'german'     => 'de',
-        'hausa'      => 'ha',
-        'hawaiian'   => null,
-        'hindi'      => 'hi',
-        'hungarian'  => 'hu',
-        'icelandic'  => 'is',
-        'indonesian' => 'id',
-        'italian'    => 'it',
-        'kazakh'     => 'kk',
-        'kyrgyz'     => 'ky',
-        'latin'      => 'la',
-        'latvian'    => 'lv',
-        'lithuanian' => 'lt',
-        'macedonian' => 'mk',
-        'mongolian'  => 'mn',
-        'nepali'     => 'ne',
-        'norwegian'  => 'no',
-        'pashto'     => 'ps',
-        'pidgin'     => null,
-        'polish'     => 'pl',
-        'portuguese' => 'pt',
-        'romanian'   => 'ro',
-        'russian'    => 'ru',
-        'serbian'    => 'sr',
-        'slovak'     => 'sk',
-        'slovene'    => 'sl',
-        'somali'     => 'so',
-        'spanish'    => 'es',
-        'swahili'    => 'sw',
-        'swedish'    => 'sv',
-        'tagalog'    => 'tl',
-        'turkish'    => 'tr',
-        'ukrainian'  => 'uk',
-        'urdu'       => 'ur',
-        'uzbek'      => 'uz',
-        'vietnamese' => 'vi',
-        'welsh'      => 'cy',
-    );
-
-    /**
-     * Maps all language names from the language database to the
-     * ISO 639-2 3-letter language code.
-     *
-     * @var array
-     */
-    public static $nameToCode3 = array(
-        'albanian'   => 'sqi',
-        'arabic'     => 'ara',
-        'azeri'      => 'aze',
-        'bengali'    => 'ben',
-        'bulgarian'  => 'bul',
-        'cebuano'    => 'ceb',
-        'croatian'   => 'hrv',
-        'czech'      => 'ces',
-        'danish'     => 'dan',
-        'dutch'      => 'nld',
-        'english'    => 'eng',
-        'estonian'   => 'est',
-        'farsi'      => 'fas',
-        'finnish'    => 'fin',
-        'french'     => 'fra',
-        'german'     => 'deu',
-        'hausa'      => 'hau',
-        'hawaiian'   => 'haw',
-        'hindi'      => 'hin',
-        'hungarian'  => 'hun',
-        'icelandic'  => 'isl',
-        'indonesian' => 'ind',
-        'italian'    => 'ita',
-        'kazakh'     => 'kaz',
-        'kyrgyz'     => 'kir',
-        'latin'      => 'lat',
-        'latvian'    => 'lav',
-        'lithuanian' => 'lit',
-        'macedonian' => 'mkd',
-        'mongolian'  => 'mon',
-        'nepali'     => 'nep',
-        'norwegian'  => 'nor',
-        'pashto'     => 'pus',
-        'pidgin'     => 'crp',
-        'polish'     => 'pol',
-        'portuguese' => 'por',
-        'romanian'   => 'ron',
-        'russian'    => 'rus',
-        'serbian'    => 'srp',
-        'slovak'     => 'slk',
-        'slovene'    => 'slv',
-        'somali'     => 'som',
-        'spanish'    => 'spa',
-        'swahili'    => 'swa',
-        'swedish'    => 'swe',
-        'tagalog'    => 'tgl',
-        'turkish'    => 'tur',
-        'ukrainian'  => 'ukr',
-        'urdu'       => 'urd',
-        'uzbek'      => 'uzb',
-        'vietnamese' => 'vie',
-        'welsh'      => 'cym',
-    );
-
-    /**
-     * Maps ISO 639-1 2-letter language codes to the language names
-     * in the language database
-     *
-     * Not all languages have a 2 letter code, so some are missing
-     *
-     * @var array
-     */
-    public static $code2ToName = array(
-        'ar' => 'arabic',
-        'az' => 'azeri',
-        'bg' => 'bulgarian',
-        'bn' => 'bengali',
-        'cs' => 'czech',
-        'cy' => 'welsh',
-        'da' => 'danish',
-        'de' => 'german',
-        'en' => 'english',
-        'es' => 'spanish',
-        'et' => 'estonian',
-        'fa' => 'farsi',
-        'fi' => 'finnish',
-        'fr' => 'french',
-        'ha' => 'hausa',
-        'hi' => 'hindi',
-        'hr' => 'croatian',
-        'hu' => 'hungarian',
-        'id' => 'indonesian',
-        'is' => 'icelandic',
-        'it' => 'italian',
-        'kk' => 'kazakh',
-        'ky' => 'kyrgyz',
-        'la' => 'latin',
-        'lt' => 'lithuanian',
-        'lv' => 'latvian',
-        'mk' => 'macedonian',
-        'mn' => 'mongolian',
-        'ne' => 'nepali',
-        'nl' => 'dutch',
-        'no' => 'norwegian',
-        'pl' => 'polish',
-        'ps' => 'pashto',
-        'pt' => 'portuguese',
-        'ro' => 'romanian',
-        'ru' => 'russian',
-        'sk' => 'slovak',
-        'sl' => 'slovene',
-        'so' => 'somali',
-        'sq' => 'albanian',
-        'sr' => 'serbian',
-        'sv' => 'swedish',
-        'sw' => 'swahili',
-        'tl' => 'tagalog',
-        'tr' => 'turkish',
-        'uk' => 'ukrainian',
-        'ur' => 'urdu',
-        'uz' => 'uzbek',
-        'vi' => 'vietnamese',
-    );
-
-    /**
-     * Maps ISO 639-2 3-letter language codes to the language names
-     * in the language database.
-     *
-     * @var array
-     */
-    public static $code3ToName = array(
-        'ara' => 'arabic',
-        'aze' => 'azeri',
-        'ben' => 'bengali',
-        'bul' => 'bulgarian',
-        'ceb' => 'cebuano',
-        'ces' => 'czech',
-        'crp' => 'pidgin',
-        'cym' => 'welsh',
-        'dan' => 'danish',
-        'deu' => 'german',
-        'eng' => 'english',
-        'est' => 'estonian',
-        'fas' => 'farsi',
-        'fin' => 'finnish',
-        'fra' => 'french',
-        'hau' => 'hausa',
-        'haw' => 'hawaiian',
-        'hin' => 'hindi',
-        'hrv' => 'croatian',
-        'hun' => 'hungarian',
-        'ind' => 'indonesian',
-        'isl' => 'icelandic',
-        'ita' => 'italian',
-        'kaz' => 'kazakh',
-        'kir' => 'kyrgyz',
-        'lat' => 'latin',
-        'lav' => 'latvian',
-        'lit' => 'lithuanian',
-        'mkd' => 'macedonian',
-        'mon' => 'mongolian',
-        'nep' => 'nepali',
-        'nld' => 'dutch',
-        'nor' => 'norwegian',
-        'pol' => 'polish',
-        'por' => 'portuguese',
-        'pus' => 'pashto',
-        'rom' => 'romanian',
-        'rus' => 'russian',
-        'slk' => 'slovak',
-        'slv' => 'slovene',
-        'som' => 'somali',
-        'spa' => 'spanish',
-        'sqi' => 'albanian',
-        'srp' => 'serbian',
-        'swa' => 'swahili',
-        'swe' => 'swedish',
-        'tgl' => 'tagalog',
-        'tur' => 'turkish',
-        'ukr' => 'ukrainian',
-        'urd' => 'urdu',
-        'uzb' => 'uzbek',
-        'vie' => 'vietnamese',
-    );
-
-    /**
-     * Returns the 2-letter ISO 639-1 code for the given language name.
-     *
-     * @param string $lang English language name like "swedish"
-     *
-     * @return string Two-letter language code (e.g. "sv") or NULL if not found
-     */
-    public static function nameToCode2($lang)
-    {
-        $lang = strtolower($lang);
-        if (!isset(self::$nameToCode2[$lang])) {
-            return null;
-        }
-        return self::$nameToCode2[$lang];
-    }
-
-    /**
-     * Returns the 3-letter ISO 639-2 code for the given language name.
-     *
-     * @param string $lang English language name like "swedish"
-     *
-     * @return string Three-letter language code (e.g. "swe") or NULL if not found
-     */
-    public static function nameToCode3($lang)
-    {
-        $lang = strtolower($lang);
-        if (!isset(self::$nameToCode3[$lang])) {
-            return null;
-        }
-        return self::$nameToCode3[$lang];
-    }
-
-    /**
-     * Returns the language name for the given 2-letter ISO 639-1 code.
-     *
-     * @param string $code Two-letter language code (e.g. "sv")
-     *
-     * @return string English language name like "swedish"
-     */
-    public static function code2ToName($code)
-    {
-        $lang = strtolower($code);
-        if (!isset(self::$code2ToName[$code])) {
-            return null;
-        }
-        return self::$code2ToName[$code];
-    }
-
-    /**
-     * Returns the language name for the given 3-letter ISO 639-2 code.
-     *
-     * @param string $code Three-letter language code (e.g. "swe")
-     *
-     * @return string English language name like "swedish"
-     */
-    public static function code3ToName($code)
-    {
-        $lang = strtolower($code);
-        if (!isset(self::$code3ToName[$code])) {
-            return null;
-        }
-        return self::$code3ToName[$code];
-    }
-}
-
-?>
\ No newline at end of file
diff --git a/library/langdet/Text/LanguageDetect/Parser.php b/library/langdet/Text/LanguageDetect/Parser.php
deleted file mode 100644
index 1c20c2657..000000000
--- a/library/langdet/Text/LanguageDetect/Parser.php
+++ /dev/null
@@ -1,349 +0,0 @@
-<?php
-
-/**
- * This class represents a text sample to be parsed.
- *
- * @category    Text
- * @package     Text_LanguageDetect
- * @author      Nicholas Pisarro
- * @copyright   2006
- * @license     BSD
- * @version     CVS: $Id: Parser.php 322327 2012-01-15 17:55:59Z cweiske $
- * @link        http://pear.php.net/package/Text_LanguageDetect/
- * @link        http://langdetect.blogspot.com/
- */
-
-/**
- * This class represents a text sample to be parsed.
- *
- * This separates the analysis of a text sample from the primary LanguageDetect
- * class. After a new profile has been built, the data can be retrieved using
- * the accessor functions.
- *
- * This class is intended to be used by the Text_LanguageDetect class, not 
- * end-users.
- *
- * @category    Text
- * @package     Text_LanguageDetect
- * @author      Nicholas Pisarro
- * @copyright   2006
- * @license     BSD
- * @version     release: 0.3.0
- */
-class Text_LanguageDetect_Parser extends Text_LanguageDetect
-{
-    /**
-     * the piece of text being parsed
-     *
-     * @access  private
-     * @var     string
-     */
-    var $_string;
-
-    /**
-     * stores the trigram frequencies of the sample
-     *
-     * @access  private
-     * @var     string
-     */
-    var $_trigrams = array();
-
-    /**
-     * stores the trigram ranks of the sample
-     *
-     * @access  private
-     * @var     array
-     */
-    var $_trigram_ranks = array();
-
-    /**
-     * stores the unicode blocks of the sample
-     *
-     * @access  private
-     * @var     array
-     */
-    var $_unicode_blocks = array();
-    
-    /**
-     * Whether the parser should compile the unicode ranges
-     * 
-     * @access  private
-     * @var     bool
-     */
-    var $_compile_unicode = false;
-
-    /**
-     * Whether the parser should compile trigrams
-     *
-     * @access  private
-     * @var     bool
-     */
-    var $_compile_trigram = false;
-
-    /**
-     * Whether the trigram parser should pad the beginning of the string
-     *
-     * @access  private
-     * @var     bool
-     */
-    var $_trigram_pad_start = false;
-
-    /**
-     * Whether the unicode parser should skip non-alphabetical ascii chars
-     *
-     * @access  private
-     * @var     bool
-     */
-    var $_unicode_skip_symbols = true;
-
-    /**
-     * Constructor
-     *
-     * @access  private
-     * @param   string  $string     string to be parsed
-     */
-    function Text_LanguageDetect_Parser($string) {
-        $this->_string = $string;
-    }
-
-    /**
-     * Returns true if a string is suitable for parsing
-     *
-     * @param   string  $str    input string to test
-     * @return  bool            true if acceptable, false if not
-     */
-    public static function validateString($str) {
-        if (!empty($str) && strlen($str) > 3 && preg_match('/\S/', $str)) {
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    /**
-     * turn on/off trigram counting
-     *
-     * @access  public
-     * @param   bool    $bool true for on, false for off
-     */
-    function prepareTrigram($bool = true)
-    {
-        $this->_compile_trigram = $bool;
-    }
-
-    /**
-     * turn on/off unicode block counting
-     *
-     * @access  public
-     * @param   bool    $bool true for on, false for off
-     */
-    function prepareUnicode($bool = true)
-    {
-        $this->_compile_unicode = $bool;
-    }
-
-    /**
-     * turn on/off padding the beginning of the sample string
-     *
-     * @access  public
-     * @param   bool    $bool true for on, false for off
-     */
-    function setPadStart($bool = true)
-    {
-        $this->_trigram_pad_start = $bool;
-    }
-
-    /**
-     * Should the unicode block counter skip non-alphabetical ascii chars?
-     *
-     * @access  public
-     * @param   bool    $bool true for on, false for off
-     */
-    function setUnicodeSkipSymbols($bool = true)
-    {
-        $this->_unicode_skip_symbols = $bool;
-    }
-
-    /**
-     * Returns the trigram ranks for the text sample
-     *
-     * @access  public
-     * @return  array    trigram ranks in the text sample
-     */
-    function &getTrigramRanks()
-    {
-        return $this->_trigram_ranks;
-    }
-
-    /**
-     * Return the trigram freqency table
-     *
-     * only used in testing to make sure the parser is working
-     *
-     * @access  public
-     * @return  array    trigram freqencies in the text sample
-     */
-    function &getTrigramFreqs()
-    {
-        return $this->_trigram;
-    }
-
-    /**
-     * returns the array of unicode blocks
-     *
-     * @access  public
-     * @return  array   unicode blocks in the text sample
-     */
-    function &getUnicodeBlocks()
-    {
-        return $this->_unicode_blocks;
-    }
-
-    /**
-     * Executes the parsing operation
-     * 
-     * Be sure to call the set*() functions to set options and the 
-     * prepare*() functions first to tell it what kind of data to compute
-     *
-     * Afterwards the get*() functions can be used to access the compiled
-     * information.
-     *
-     * @access public
-     */
-    function analyze()
-    {
-        $len = strlen($this->_string);
-        $byte_counter = 0;
-
-
-        // unicode startup
-        if ($this->_compile_unicode) {
-            $blocks = $this->_read_unicode_block_db();
-            $block_count = count($blocks);
-
-            $skipped_count = 0;
-            $unicode_chars = array();
-        }
-
-        // trigram startup
-        if ($this->_compile_trigram) {
-            // initialize them as blank so the parser will skip the first two
-            // (since it skips trigrams with more than  2 contiguous spaces)
-            $a = ' ';
-            $b = ' ';
-
-            // kludge
-            // if it finds a valid trigram to start and the start pad option is
-            // off, then set a variable that will be used to reduce this
-            // trigram after parsing has finished
-            if (!$this->_trigram_pad_start) {
-                $a = $this->_next_char($this->_string, $byte_counter, true);
-
-                if ($a != ' ') {
-                    $b = $this->_next_char($this->_string, $byte_counter, true);
-                    $dropone = " $a$b";
-                }
-
-                $byte_counter = 0;
-                $a = ' ';
-                $b = ' ';
-            }
-        }
-
-        while ($byte_counter < $len) {
-            $char = $this->_next_char($this->_string, $byte_counter, true);
-
-
-            // language trigram detection
-            if ($this->_compile_trigram) {
-                if (!($b == ' ' && ($a == ' ' || $char == ' '))) {
-                    if (!isset($this->_trigram[$a . $b . $char])) {
-                       $this->_trigram[$a . $b . $char] = 1;
-                    } else {
-                       $this->_trigram[$a . $b . $char]++;
-                    }
-                }
-
-                $a = $b;
-                $b = $char;
-            }
-
-            // unicode block detection
-            if ($this->_compile_unicode) {
-                if ($this->_unicode_skip_symbols
-                        && strlen($char) == 1
-                        && ($char < 'A' || $char > 'z'
-                        || ($char > 'Z' && $char < 'a'))
-                        && $char != "'") {  // does not skip the apostrophe
-                                            // since it's included in the language
-                                            // models
-
-                    $skipped_count++;
-                    continue;
-                }
-
-                // build an array of all the characters
-                if (isset($unicode_chars[$char])) {
-                    $unicode_chars[$char]++;
-                } else {
-                    $unicode_chars[$char] = 1;
-                }
-            }
-
-            // todo: add byte detection here
-        }
-
-        // unicode cleanup
-        if ($this->_compile_unicode) {
-            foreach ($unicode_chars as $utf8_char => $count) {
-                $search_result = $this->_unicode_block_name(
-                        $this->_utf8char2unicode($utf8_char), $blocks, $block_count);
-
-                if ($search_result != -1) {
-                    $block_name = $search_result[2];
-                } else {
-                    $block_name = '[Malformatted]';
-                }
-
-                if (isset($this->_unicode_blocks[$block_name])) {
-                    $this->_unicode_blocks[$block_name] += $count;
-                } else {
-                    $this->_unicode_blocks[$block_name] = $count;
-                }
-            }
-        }
-
-
-        // trigram cleanup
-        if ($this->_compile_trigram) {
-            // pad the end
-            if ($b != ' ') {
-                if (!isset($this->_trigram["$a$b "])) {
-                    $this->_trigram["$a$b "] = 1;
-                } else {
-                    $this->_trigram["$a$b "]++;
-                }
-            }
-
-            // perl compatibility; Language::Guess does not pad the beginning
-            // kludge
-            if (isset($dropone)) {
-                if ($this->_trigram[$dropone] == 1) {
-                    unset($this->_trigram[$dropone]);
-                } else {
-                    $this->_trigram[$dropone]--;
-                }
-            }
-
-            if (!empty($this->_trigram)) {
-                $this->_trigram_ranks = $this->_arr_rank($this->_trigram);
-            } else {
-                $this->_trigram_ranks = array();
-            }
-        }
-    }
-}
-
-/* vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4: */
-
-?>