aboutsummaryrefslogtreecommitdiffstats
path: root/library/spam/b8/b8.php.ORIG
diff options
context:
space:
mode:
Diffstat (limited to 'library/spam/b8/b8.php.ORIG')
-rw-r--r--library/spam/b8/b8.php.ORIG503
1 files changed, 503 insertions, 0 deletions
diff --git a/library/spam/b8/b8.php.ORIG b/library/spam/b8/b8.php.ORIG
new file mode 100644
index 000000000..ea1e15ffa
--- /dev/null
+++ b/library/spam/b8/b8.php.ORIG
@@ -0,0 +1,503 @@
+<?php
+
+# Copyright (C) 2006-2010 Tobias Leupold <tobias.leupold@web.de>
+#
+# b8 - A Bayesian spam filter written in PHP 5
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation in version 2.1 of the License.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+# License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
+
+/**
+ * Copyright (C) 2006-2010 Tobias Leupold <tobias.leupold@web.de>
+ *
+ * @license LGPL
+ * @access public
+ * @package b8
+ * @author Tobias Leupold
+ * @author Oliver Lillie (aka buggedcom) (original PHP 5 port)
+ */
+
+class b8
+{
+
+ public $config = array(
+ 'min_size' => 3,
+ 'max_size' => 30,
+ 'allow_numbers' => FALSE,
+ 'lexer' => 'default',
+ 'degenerator' => 'default',
+ 'storage' => 'dba',
+ 'use_relevant' => 15,
+ 'min_dev' => 0.2,
+ 'rob_s' => 0.3,
+ 'rob_x' => 0.5
+ );
+
+ private $_lexer = NULL;
+ private $_database = NULL;
+ private $_token_data = NULL;
+
+ const SPAM = 'spam';
+ const HAM = 'ham';
+ const LEARN = 'learn';
+ const UNLEARN = 'unlearn';
+
+ const STARTUP_FAIL_DATABASE = 'STARTUP_FAIL_DATABASE';
+ const STARTUP_FAIL_LEXER = 'STARTUP_FAIL_LEXER';
+ const TRAINER_CATEGORY_FAIL = 'TRAINER_CATEGORY_FAIL';
+
+ /**
+ * Constructs b8
+ *
+ * @access public
+ * @return void
+ */
+
+ function __construct($config = array(), $database_config)
+ {
+
+ # Validate config data
+
+ if(count($config) > 0) {
+
+ foreach ($config as $name=>$value) {
+
+ switch($name) {
+
+ case 'min_dev':
+ case 'rob_s':
+ case 'rob_x':
+ $this->config[$name] = (float) $value;
+ break;
+
+ case 'min_size':
+ case 'max_size':
+ case 'use_relevant':
+ $this->config[$name] = (int) $value;
+ break;
+
+ case 'allow_numbers':
+ $this->config[$name] = (bool) $value;
+ break;
+
+ case 'lexer':
+ $value = (string) strtolower($value);
+ $this->config[$name] = is_file(dirname(__FILE__) . DIRECTORY_SEPARATOR . 'lexer' . DIRECTORY_SEPARATOR . "lexer_" . $value . '.php') === TRUE ? $value : 'default';
+ break;
+
+ case 'storage':
+ $this->config[$name] = (string) $value;
+ break;
+
+ }
+
+ }
+
+ }
+
+ # Setup the database backend
+
+ # Get the basic storage class used by all backends
+ if($this->load_class('b8_storage_base', dirname(__FILE__) . DIRECTORY_SEPARATOR . 'storage' . DIRECTORY_SEPARATOR . 'storage_base.php') === FALSE)
+ return;
+
+ # Get the degenerator we need
+ if($this->load_class('b8_degenerator_' . $this->config['degenerator'], dirname(__FILE__) . DIRECTORY_SEPARATOR . 'degenerator' . DIRECTORY_SEPARATOR . 'degenerator_' . $this->config['degenerator'] . '.php') === FALSE)
+ return;
+
+ # Get the actual storage backend we need
+ if($this->load_class('b8_storage_' . $this->config['storage'], dirname(__FILE__) . DIRECTORY_SEPARATOR . 'storage' . DIRECTORY_SEPARATOR . 'storage_' . $this->config['storage'] . '.php') === FALSE)
+ return;
+
+ # Setup the backend
+ $class = 'b8_storage_' . $this->config['storage'];
+ $this->_database = new $class(
+ $database_config,
+ $this->config['degenerator'], date('ymd')
+ );
+
+ # Setup the lexer class
+
+ if($this->load_class('b8_lexer_' . $this->config['lexer'], dirname(__FILE__) . DIRECTORY_SEPARATOR . 'lexer' . DIRECTORY_SEPARATOR . 'lexer_' . $this->config['lexer'] . '.php') === FALSE)
+ return;
+
+ $class = 'b8_lexer_' . $this->config['lexer'];
+ $this->_lexer = new $class(
+ array(
+ 'min_size' => $this->config['min_size'],
+ 'max_size' => $this->config['max_size'],
+ 'allow_numbers' => $this->config['allow_numbers']
+ )
+ );
+
+ }
+
+ /**
+ * Load a class file if a class has not been defined yet.
+ *
+ * @access public
+ * @return boolean Returns TRUE if everything is okay, otherwise FALSE.
+ */
+
+ public function load_class($class_name, $class_file)
+ {
+
+ if(class_exists($class_name, FALSE) === FALSE) {
+
+ $included = require_once $class_file;
+
+ if($included === FALSE or class_exists($class_name, FALSE) === FALSE)
+ return FALSE;
+
+ }
+
+ return TRUE;
+
+ }
+
+ /**
+ * Validates the class has all it needs to work.
+ *
+ * @access public
+ * @return mixed Returns TRUE if everything is okay, otherwise an error code.
+ */
+
+ public function validate()
+ {
+
+ if($this->_database === NULL)
+ return self::STARTUP_FAIL_DATABASE;
+
+ # Connect the database backend if we aren't connected yet
+
+ elseif($this->_database->connected === FALSE) {
+
+ $connection = $this->_database->connect();
+
+ if($connection !== TRUE)
+ return $connection;
+
+ }
+
+ if($this->_lexer === NULL)
+ return self::STARTUP_FAIL_LEXER;
+
+ return TRUE;
+
+ }
+
+ /**
+ * Classifies a text
+ *
+ * @access public
+ * @package default
+ * @param string $text
+ * @return float The rating between 0 (ham) and 1 (spam)
+ */
+
+ public function classify($text)
+ {
+
+ # Validate the startup
+
+ $started_up = $this->validate();
+
+ if($started_up !== TRUE)
+ return $started_up;
+
+ # Get the internal database variables, containing the number of ham and
+ # spam texts so the spam probability can be calculated in relation to them
+ $internals = $this->_database->get_internals();
+
+ # Calculate the spamminess of all tokens
+
+ # Get all tokens we want to rate
+
+ $tokens = $this->_lexer->get_tokens($text);
+
+ # Check if the lexer failed
+ # (if so, $tokens will be a lexer error code, if not, $tokens will be an array)
+ if(!is_array($tokens))
+ return $tokens;
+
+ # Fetch all availible data for the token set from the database
+ $this->_token_data = $this->_database->get(array_keys($tokens));
+
+ # Calculate the spamminess and importance for each token (or a degenerated form of it)
+
+ $word_count = array();
+ $rating = array();
+ $importance = array();
+
+ foreach($tokens as $word => $count) {
+
+ $word_count[$word] = $count;
+
+ # Although we only call this function only here ... let's do the
+ # calculation stuff in a function to make this a bit less confusing ;-)
+ $rating[$word] = $this->_get_probability($word, $internals['texts_ham'], $internals['texts_spam']);
+
+ $importance[$word] = abs(0.5 - $rating[$word]);
+
+ }
+
+ # Order by importance
+ arsort($importance);
+ reset($importance);
+
+ # Get the most interesting tokens (use all if we have less than the given number)
+
+ $relevant = array();
+
+ for($i = 0; $i < $this->config['use_relevant']; $i++) {
+
+ if($tmp = each($importance)) {
+
+ # Important tokens remain
+
+ # If the token's rating is relevant enough, use it
+
+ if(abs(0.5 - $rating[$tmp['key']]) > $this->config['min_dev']) {
+
+ # Tokens that appear more than once also count more than once
+
+ for($x = 0, $l = $word_count[$tmp['key']]; $x < $l; $x++)
+ array_push($relevant, $rating[$tmp['key']]);
+
+ }
+
+ }
+
+ else {
+ # We have less than words to use, so we already
+ # use what we have and can break here
+ break;
+ }
+
+ }
+
+ # Calculate the spamminess of the text (thanks to Mr. Robinson ;-)
+ # We set both hamminess and Spamminess to 1 for the first multiplying
+ $hamminess = 1;
+ $spamminess = 1;
+
+ # Consider all relevant ratings
+ foreach($relevant as $value) {
+ $hamminess *= (1.0 - $value);
+ $spamminess *= $value;
+ }
+
+ # If no token was good for calculation, we really don't know how
+ # to rate this text; so we assume a spam and ham probability of 0.5
+
+ if($hamminess === 1 and $spamminess === 1) {
+ $hamminess = 0.5;
+ $spamminess = 0.5;
+ $n = 1;
+ }
+ else {
+ # Get the number of relevant ratings
+ $n = count($relevant);
+ }
+
+ # Calculate the combined rating
+
+ # The actual hamminess and spamminess
+ $hamminess = 1 - pow($hamminess, (1 / $n));
+ $spamminess = 1 - pow($spamminess, (1 / $n));
+
+ # Calculate the combined indicator
+ $probability = ($hamminess - $spamminess) / ($hamminess + $spamminess);
+
+ # We want a value between 0 and 1, not between -1 and +1, so ...
+ $probability = (1 + $probability) / 2;
+
+ # Alea iacta est
+ return $probability;
+
+ }
+
+ /**
+ * Calculate the spamminess of a single token also considering "degenerated" versions
+ *
+ * @access private
+ * @param string $word
+ * @param string $texts_ham
+ * @param string $texts_spam
+ * @return void
+ */
+
+ private function _get_probability($word, $texts_ham, $texts_spam)
+ {
+
+ # Let's see what we have!
+
+ if(isset($this->_token_data['tokens'][$word]) === TRUE) {
+ # The token was in the database, so we can use it's data as-is
+ # and calculate the spamminess of this token directly
+ return $this->_calc_probability($this->_token_data['tokens'][$word], $texts_ham, $texts_spam);
+ }
+
+ # Damn. The token was not found, so do we have at least similar words?
+
+ if(isset($this->_token_data['degenerates'][$word]) === TRUE) {
+
+ # We found similar words, so calculate the spamminess for each one
+ # and choose the most important one for the further calculation
+
+ # The default rating is 0.5 simply saying nothing
+ $rating = 0.5;
+
+ foreach($this->_token_data['degenerates'][$word] as $degenerate => $count) {
+
+ # Calculate the rating of the current degenerated token
+ $rating_tmp = $this->_calc_probability($count, $texts_ham, $texts_spam);
+
+ # Is it more important than the rating of another degenerated version?
+ if(abs(0.5 - $rating_tmp) > abs(0.5 - $rating))
+ $rating = $rating_tmp;
+
+ }
+
+ return $rating;
+
+ }
+
+ else {
+ # The token is really unknown, so choose the default rating
+ # for completely unknown tokens. This strips down to the
+ # robX parameter so we can cheap out the freaky math ;-)
+ return $this->config['rob_x'];
+ }
+
+ }
+
+ /**
+ * Do the actual spamminess calculation of a single token
+ *
+ * @access private
+ * @param array $data
+ * @param string $texts_ham
+ * @param string $texts_spam
+ * @return void
+ */
+
+ private function _calc_probability($data, $texts_ham, $texts_spam)
+ {
+
+ # Calculate the basic probability by Mr. Graham
+
+ # But: consider the number of ham and spam texts saved instead of the
+ # number of entries where the token appeared to calculate a relative
+ # spamminess because we count tokens appearing multiple times not just
+ # once but as often as they appear in the learned texts
+
+ $rel_ham = $data['count_ham'];
+ $rel_spam = $data['count_spam'];
+
+ if($texts_ham > 0)
+ $rel_ham = $data['count_ham'] / $texts_ham;
+
+ if($texts_spam > 0)
+ $rel_spam = $data['count_spam'] / $texts_spam;
+
+ $rating = $rel_spam / ($rel_ham + $rel_spam);
+
+ # Calculate the better probability proposed by Mr. Robinson
+ $all = $data['count_ham'] + $data['count_spam'];
+ return (($this->config['rob_s'] * $this->config['rob_x']) + ($all * $rating)) / ($this->config['rob_s'] + $all);
+
+ }
+
+ /**
+ * Check the validity of the category of a request
+ *
+ * @access private
+ * @param string $category
+ * @return void
+ */
+
+ private function _check_category($category)
+ {
+ return $category === self::HAM or $category === self::SPAM;
+ }
+
+ /**
+ * Learn a reference text
+ *
+ * @access public
+ * @param string $text
+ * @param const $category Either b8::SPAM or b8::HAM
+ * @return void
+ */
+
+ public function learn($text, $category)
+ {
+ return $this->_process_text($text, $category, self::LEARN);
+ }
+
+ /**
+ * Unlearn a reference text
+ *
+ * @access public
+ * @param string $text
+ * @param const $category Either b8::SPAM or b8::HAM
+ * @return void
+ */
+
+ public function unlearn($text, $category)
+ {
+ return $this->_process_text($text, $category, self::UNLEARN);
+ }
+
+ /**
+ * Does the actual interaction with the storage backend for learning or unlearning texts
+ *
+ * @access private
+ * @param string $text
+ * @param const $category Either b8::SPAM or b8::HAM
+ * @param const $action Either b8::LEARN or b8::UNLEARN
+ * @return void
+ */
+
+ private function _process_text($text, $category, $action)
+ {
+
+ # Validate the startup
+
+ $started_up = $this->validate();
+
+ if($started_up !== TRUE)
+ return $started_up;
+
+ # Look if the request is okay
+ if($this->_check_category($category) === FALSE)
+ return self::TRAINER_CATEGORY_FAIL;
+
+ # Get all tokens from $text
+
+ $tokens = $this->_lexer->get_tokens($text);
+
+ # Check if the lexer failed
+ # (if so, $tokens will be a lexer error code, if not, $tokens will be an array)
+ if(!is_array($tokens))
+ return $tokens;
+
+ # Pass the tokens and what to do with it to the storage backend
+ return $this->_database->process_text($tokens, $category, $action);
+
+ }
+
+}
+
+?> \ No newline at end of file
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-08 19:24:25 +0000