1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
|
<?php
# Copyright (C) 2006-2010 Tobias Leupold <tobias.leupold@web.de>
#
# This file is part of the b8 package
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation in version 2.1 of the License.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
# License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.
/**
* Copyright (C) 2006-2010 Tobias Leupold <tobias.leupold@web.de>
*
* @license LGPL
* @access public
* @package b8
* @author Tobias Leupold
* @author Oliver Lillie (aka buggedcom) (original PHP 5 port)
*/
class b8_lexer_default
{
const LEXER_TEXT_NOT_STRING = 'LEXER_TEXT_NOT_STRING';
const LEXER_TEXT_EMPTY = 'LEXER_TEXT_EMPTY';
public $config = NULL;
# The regular expressions we use to split the text to tokens
public $regexp = array(
'ip' => '/([A-Za-z0-9\_\-\.]+)/',
'raw_split' => '/[\s,\.\/"\:;\|<>\-_\[\]{}\+=\)\(\*\&\^%]+/',
'html' => '/(<.+?>)/',
'tagname' => '/(.+?)\s/',
'numbers' => '/^[0-9]+$/'
);
/**
* Constructs the lexer.
*
* @access public
* @return void
*/
function __construct($config)
{
$this->config = $config;
}
/**
* Generates the tokens required for the bayesian filter.
*
* @access public
* @param string $text
* @return array Returns the list of tokens
*/
public function get_tokens($text)
{
# Check that we actually have a string ...
if(is_string($text) === FALSE)
return self::LEXER_TEXT_NOT_STRING;
# ... and that it's not empty
if(empty($text) === TRUE)
return self::LEXER_TEXT_EMPTY;
# Re-convert the text to the original characters coded in UTF-8, as
# they have been coded in html entities during the post process
$text = html_entity_decode($text, ENT_QUOTES, 'UTF-8');
$tokens = array();
# Find URLs and IP addresses
preg_match_all($this->regexp['ip'], $text, $raw_tokens);
foreach($raw_tokens[1] as $word) {
# Check for a dot
if(strpos($word, '.') === FALSE)
continue;
# Check that the word is valid, min and max sizes, etc.
if($this->_is_valid($word) === FALSE)
continue;
if(isset($tokens[$word]) === FALSE)
$tokens[$word] = 1;
else
$tokens[$word] += 1;
# Delete the word from the text so it doesn't get re-added.
$text = str_replace($word, '', $text);
# Also process the parts of the URLs
$url_parts = preg_split($this->regexp['raw_split'], $word);
foreach($url_parts as $word) {
# Again validate the part
if($this->_is_valid($word) === FALSE)
continue;
if(isset($tokens[$word]) === FALSE)
$tokens[$word] = 1;
else
$tokens[$word] += 1;
}
}
# Split the remaining text
$raw_tokens = preg_split($this->regexp['raw_split'], $text);
foreach($raw_tokens as $word) {
# Again validate the part
if($this->_is_valid($word) === FALSE)
continue;
if(isset($tokens[$word]) === FALSE)
$tokens[$word] = 1;
else
$tokens[$word] += 1;
}
# Process the HTML
preg_match_all($this->regexp['html'], $text, $raw_tokens);
foreach($raw_tokens[1] as $word) {
# Again validate the part
if($this->_is_valid($word) === FALSE)
continue;
# If the tag has parameters, just use the tag itself
if(strpos($word, ' ') !== FALSE) {
preg_match($this->regexp['tagname'], $word, $tmp);
$word = "{$tmp[1]}...>";
}
if(isset($tokens[$word]) === FALSE)
$tokens[$word] = 1;
else
$tokens[$word] += 1;
}
# Return a list of all found tokens
return $tokens;
}
/**
* Validates a token.
*
* @access private
* @param string $token The token string.
* @return boolean Returns TRUE if the token is valid, otherwise returns FALSE
*/
private function _is_valid($token)
{
# Validate the size of the token
$len = strlen($token);
if($len < $this->config['min_size'] or $len > $this->config['max_size'])
return FALSE;
# We may want to exclude pure numbers
if($this->config['allow_numbers'] === FALSE) {
if(preg_match($this->regexp['numbers'], $token) > 0)
return FALSE;
}
# Token is okay
return TRUE;
}
}
?>
|
