enhanced content filters

5 files changed, 230 insertions, 75 deletions

diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php
index e959ac879..26f7db7b8 100644
--- a/Zotlabs/Lib/Activity.php
+++ b/Zotlabs/Lib/Activity.php
@@ -1926,11 +1926,6 @@ class Activity {
 			}
 		}
 
-		$abook = q("select * from abook where abook_xchan = '%s' and abook_channel = %d limit 1",
-			dbesc($observer_hash),
-			intval($channel['channel_id'])
-		);
-
 		$content = self::get_content($act->obj);
 
 		if (!$content) {
@@ -2014,9 +2009,14 @@ class Activity {
 			}
 		}
 
+		$abook = q("select * from abook where (abook_xchan = '%s' OR abook_xchan  = '%s') and abook_channel = %d ",
+			dbesc($s['author_xchan']),
+			dbesc($s['owner_xchan']),
+			intval($channel['channel_id'])
+		);
 
 		if ($abook) {
-			if (!post_is_importable($s, $abook[0])) {
+			if (!post_is_importable($channel['channel_id'], $s, $abook)) {
 				logger('post is filtered');
 				return;
 			}
@@ -2934,13 +2934,14 @@ class Activity {
 			}
 		}
 
-		$abook = q("select * from abook where abook_xchan = '%s' and abook_channel = %d limit 1",
-			dbesc($observer_hash),
+		$abook = q("select * from abook where ( abook_xchan = '%s' OR abook_xchan  = '%s') and abook_channel = %d ",
+			dbesc($item['author_xchan']),
+			dbesc($item['owner_xchan']),
 			intval($channel['channel_id'])
 		);
 
 		if ($abook) {
-			if (!post_is_importable($item, $abook[0])) {
+			if (!post_is_importable($channel['channel_id'], $item, $abook)) {
 				logger('post is filtered');
 				return;
 			}
@@ -3293,6 +3294,7 @@ class Activity {
 
 	}
 
+/* this is deprecated and not used anymore
 	static function announce_note($channel, $observer_hash, $act) {
 
 		$s              = [];
@@ -3423,6 +3425,7 @@ class Activity {
 		}
 
 	}
+*/
 
 	static function like_note($channel, $observer_hash, $act) {
 
diff --git a/Zotlabs/Lib/Libzot.php b/Zotlabs/Lib/Libzot.php
index 52d08e000..fdeb7a3b0 100644
--- a/Zotlabs/Lib/Libzot.php
+++ b/Zotlabs/Lib/Libzot.php
@@ -1753,11 +1753,13 @@ class Libzot {
 				}
 			}
 
-			$ab = q("select * from abook where abook_channel = %d and abook_xchan = '%s'",
+			// This is used to fetch allow/deny rules if either the sender
+			// or  owner is  a connection. post_is_importable() evaluates all of them
+			$abook = q("select * from abook where abook_channel = %d and ( abook_xchan = '%s' OR abook_xchan = '%s' )",
 				intval($channel['channel_id']),
-				dbesc($arr['owner_xchan'])
+				dbesc($arr['owner_xchan']),
+				dbesc($arr['author_xchan'])
 			);
-			$abook = (($ab) ? $ab[0] : null);
 
 			if (intval($arr['item_deleted'])) {
 
@@ -1808,17 +1810,18 @@ class Libzot {
 				elseif ($arr['edited'] > $r[0]['edited']) {
 					$arr['id']  = $r[0]['id'];
 					$arr['uid'] = $channel['channel_id'];
-					if (($arr['mid'] == $arr['parent_mid']) && (!post_is_importable($arr, $abook))) {
-						$DR->update('update ignored');
-						$result[] = $DR->get();
-					}
-					else {
-						$item_result = self::update_imported_item($sender, $arr, $r[0], $channel['channel_id'], $tag_delivery);
-						$DR->update('updated');
-						$result[] = $DR->get();
-						if (!$relay)
-							add_source_route($item_id, $sender);
-					}
+
+                    if (post_is_importable($channel['channel_id'], $arr, $abook)) {
+                        $item_result = self::update_imported_item($sender, $arr, $r[0], $channel['channel_id'], $tag_delivery);
+                        $DR->update('updated');
+                        $result[] = $DR->get();
+                        if (!$relay) {
+                            add_source_route($item_id, $sender);
+                        }
+                    } else {
+                        $DR->update('update ignored');
+                        $result[] = $DR->get();
+                    }
 				}
 				else {
 					$DR->update('update ignored');
@@ -1848,20 +1851,29 @@ class Libzot {
 
 				$item_id = 0;
 
-				if (($arr['mid'] == $arr['parent_mid']) && (!post_is_importable($arr, $abook))) {
-					$DR->update('post ignored');
-					$result[] = $DR->get();
+				$maxlen = get_max_import_size();
+
+				if ($maxlen && mb_strlen($arr['body']) > $maxlen) {
+					$arr['body'] = mb_substr($arr['body'], 0, $maxlen, 'UTF-8');
+					logger('message length exceeds max_import_size: truncated');
 				}
-				else {
+
+				if ($maxlen && mb_strlen($arr['summary']) > $maxlen) {
+					$arr['summary'] = mb_substr($arr['summary'], 0, $maxlen, 'UTF-8');
+					logger('message summary length exceeds max_import_size: truncated');
+				}
+
+				if (post_is_importable($arr['uid'], $arr, $abook)) {
 					$item_result = item_store($arr);
 					if ($item_result['success']) {
 						$item_id = $item_result['item_id'];
-						$parr    = [
+						$parr = [
 							'item_id' => $item_id,
-							'item'    => $arr,
-							'sender'  => $sender,
+							'item' => $arr,
+							'sender' => $sender,
 							'channel' => $channel
 						];
+
 						/**
 						 * @hooks activity_received
 						 *   Called when an activity (post, comment, like, etc.) has been received from a zot source.
@@ -1871,13 +1883,19 @@ class Libzot {
 						 *   * \e array \b channel
 						 */
 						call_hooks('activity_received', $parr);
+
 						// don't add a source route if it's a relay or later recipients will get a route mismatch
-						if (!$relay)
+						if (!$relay) {
 							add_source_route($item_id, $sender);
+						}
 					}
 					$DR->update(($item_id) ? 'posted' : 'storage failed: ' . $item_result['message']);
 					$result[] = $DR->get();
+				} else {
+					$DR->update('post ignored');
+					$result[] = $DR->get();
 				}
+
 			}
 
 			// preserve conversations with which you are involved from expiration
diff --git a/Zotlabs/Lib/MessageFilter.php b/Zotlabs/Lib/MessageFilter.php
index 21e6ca26a..95721e7c7 100644
--- a/Zotlabs/Lib/MessageFilter.php
+++ b/Zotlabs/Lib/MessageFilter.php
@@ -2,85 +2,104 @@
 
 namespace Zotlabs\Lib;
 
-
+require_once('include/html2plain.php');
 
 class MessageFilter {
 
+	public static function evaluate($item, $incl, $excl) {
 
-	static public function evaluate($item,$incl,$excl) {
-
-		require_once('include/html2plain.php');
-
-		$text = prepare_text($item['body'],$item['mimetype']);
+		$text = prepare_text($item['body'],((isset($item['mimetype'])) ? $item['mimetype'] : 'text/x-multicode'));
 		$text = html2plain(($item['title']) ? $item['title'] . ' ' . $text : $text);
 
-
 		$lang = null;
 
-		if((strpos($incl,'lang=') !== false) || (strpos($excl,'lang=') !== false) || (strpos($incl,'lang!=') !== false) || (strpos($excl,'lang!=') !== false)) {
+		if ((strpos($incl, 'lang=') !== false) || (strpos($excl, 'lang=') !== false) || (strpos($incl, 'lang!=') !== false) || (strpos($excl, 'lang!=') !== false)) {
 			$lang = detect_language($text);
 		}
 
-		$tags = ((is_array($item['term']) && count($item['term'])) ? $item['term'] : false);
+		$tags = ((isset($item['term']) && is_array($item['term']) && count($item['term'])) ? $item['term'] : false);
 
 		// exclude always has priority
 
-		$exclude = (($excl) ? explode("\n",$excl) : null);
+		$exclude = (($excl) ? explode("\n", $excl) : null);
 
-		if($exclude) {
-			foreach($exclude as $word) {
+		if ($exclude) {
+			foreach ($exclude as $word) {
 				$word = trim($word);
-				if(! $word)
+				if (! $word) {
 					continue;
-				if(substr($word,0,1) === '#' && $tags) {
-					foreach($tags as $t)
-						if((($t['ttype'] == TERM_HASHTAG) || ($t['ttype'] == TERM_COMMUNITYTAG)) && (($t['term'] === substr($word,1)) || (substr($word,1) === '*')))
-							return false;
 				}
-				elseif(substr($word,0,1) === '$' && $tags) {
-					foreach($tags as $t)
-						if(($t['ttype'] == TERM_CATEGORY) && (($t['term'] === substr($word,1)) || (substr($word,1) === '*')))
+				if (substr($word, 0, 1) === '#' && $tags) {
+					foreach ($tags as $t) {
+						if ((($t['ttype'] == TERM_HASHTAG) || ($t['ttype'] == TERM_COMMUNITYTAG)) && (($t['term'] === substr($word, 1)) || (substr($word, 1) === '*'))) {
 							return false;
-				}
-				elseif((strpos($word,'/') === 0) && preg_match($word,$text))
+						}
+					}
+				} elseif (substr($word, 0, 1) === '$' && $tags) {
+					foreach ($tags as $t) {
+						if (($t['ttype'] == TERM_CATEGORY) && (($t['term'] === substr($word, 1)) || (substr($word, 1) === '*'))) {
+							return false;
+						}
+					}
+				} elseif (substr($word, 0, 2) === '?+') {
+					if (self::test_condition(substr($word, 2), $item['obj'])) {
+						return false;
+					}
+				} elseif (substr($word, 0, 1) === '?') {
+					if (self::test_condition(substr($word, 1), $item)) {
+						return false;
+					}
+				} elseif ((strpos($word, '/') === 0) && preg_match($word, $text)) {
 					return false;
-				elseif((strpos($word,'lang=') === 0) && ($lang) && (strcasecmp($lang,trim(substr($word,5))) == 0))
+				} elseif ((strpos($word, 'lang=') === 0) && ($lang) && (strcasecmp($lang, trim(substr($word, 5))) == 0)) {
 					return false;
-				elseif((strpos($word,'lang!=') === 0) && ($lang) && (strcasecmp($lang,trim(substr($word,6))) != 0))
+				} elseif ((strpos($word, 'lang!=') === 0) && ($lang) && (strcasecmp($lang, trim(substr($word, 6))) != 0)) {
 					return false;
-				elseif(stristr($text,$word) !== false)
+				} elseif (stristr($text, $word) !== false) {
 					return false;
+				}
 			}
 		}
 
-		$include = (($incl) ? explode("\n",$incl) : null);
+		$include = (($incl) ? explode("\n", $incl) : null);
 
-		if($include) {
-			foreach($include as $word) {
+		if ($include) {
+			foreach ($include as $word) {
 				$word = trim($word);
-				if(! $word)
+				if (! $word) {
 					continue;
-				if(substr($word,0,1) === '#' && $tags) {
-					foreach($tags as $t)
-						if((($t['ttype'] == TERM_HASHTAG) || ($t['ttype'] == TERM_COMMUNITYTAG)) && (($t['term'] === substr($word,1)) || (substr($word,1) === '*')))
-							return true;
 				}
-				elseif(substr($word,0,1) === '$' && $tags) {
-					foreach($tags as $t)
-						if(($t['ttype'] == TERM_CATEGORY) && (($t['term'] === substr($word,1)) || (substr($word,1) === '*')))
+				if (substr($word, 0, 1) === '#' && $tags) {
+					foreach ($tags as $t) {
+						if ((($t['ttype'] == TERM_HASHTAG) || ($t['ttype'] == TERM_COMMUNITYTAG)) && (($t['term'] === substr($word, 1)) || (substr($word, 1) === '*'))) {
 							return true;
-				}
-				elseif((strpos($word,'/') === 0) && preg_match($word,$text))
+						}
+					}
+				} elseif (substr($word, 0, 1) === '$' && $tags) {
+					foreach ($tags as $t) {
+						if (($t['ttype'] == TERM_CATEGORY) && (($t['term'] === substr($word, 1)) || (substr($word, 1) === '*'))) {
+							return true;
+						}
+					}
+				} elseif (substr($word, 0, 2) === '?+') {
+					if (self::test_condition(substr($word, 2), $item['obj'])) {
+						return true;
+					}
+				} elseif (substr($word, 0, 1) === '?') {
+					if (self::test_condition(substr($word, 1), $item)) {
+						return true;
+					}
+				} elseif ((strpos($word, '/') === 0) && preg_match($word, $text)) {
 					return true;
-				elseif((strpos($word,'lang=') === 0) && ($lang) && (strcasecmp($lang,trim(substr($word,5))) == 0))
+				} elseif ((strpos($word, 'lang=') === 0) && ($lang) && (strcasecmp($lang, trim(substr($word, 5))) == 0)) {
 					return true;
-				elseif((strpos($word,'lang!=') === 0) && ($lang) && (strcasecmp($lang,trim(substr($word,6))) != 0))
+				} elseif ((strpos($word, 'lang!=') === 0) && ($lang) && (strcasecmp($lang, trim(substr($word, 6))) != 0)) {
 					return true;
-				elseif(stristr($text,$word) !== false)
+				} elseif (stristr($text, $word) !== false) {
 					return true;
+				}
 			}
-		}
-		else {
+		} else {
 			return true;
 		}
 
@@ -88,4 +107,113 @@ class MessageFilter {
 	}
 
 
+	/**
+	 * @brief Test for Conditional Execution conditions. Shamelessly ripped off from Code/Render/Comanche
+	 *
+	 * This is extensible. The first version of variable testing supports tests of the forms:
+	 *
+	 * - ?foo ~= baz which will check if item.foo contains the string 'baz';
+	 * - ?foo == baz which will check if item.foo is the string 'baz';
+	 * - ?foo != baz which will check if item.foo is not the string 'baz';
+	 * - ?foo >= 3 which will check if item.foo is greater than or equal to 3;
+	 * - ?foo > 3 which will check if item.foo is greater than 3;
+	 * - ?foo <= 3 which will check if item.foo is less than or equal to 3;
+	 * - ?foo < 3 which will check if item.foo is less than 3;
+	 *
+	 * - ?foo {} baz which will check if 'baz' is an array element in item.foo
+	 * - ?foo {*} baz which will check if 'baz' is an array key in item.foo
+	 * - ?foo which will check for a return of a true condition for item.foo;
+	 *
+	 * The values 0, '', an empty array, and an unset value will all evaluate to false.
+	 *
+	 * @param string $s
+	 * @param array $item
+	 * @return bool
+	 */
+
+	public static function test_condition($s,$item) {
+
+		if (preg_match('/(.*?)\s\~\=\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if (stripos($x, trim($matches[2])) !== false) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)\s\=\=\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if ($x == trim($matches[2])) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)\s\!\=\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if ($x != trim($matches[2])) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)\s\>\=\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if ($x >= trim($matches[2])) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)\s\<\=\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if ($x <= trim($matches[2])) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)\s\>\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if ($x > trim($matches[2])) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)\s\>\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if ($x < trim($matches[2])) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/[\$](.*?)\s\{\}\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if (is_array($x) && in_array(trim($matches[2]), $x)) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)\s\{\*\}\s(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if (is_array($x) && array_key_exists(trim($matches[2]), $x)) {
+				return true;
+			}
+			return false;
+		}
+
+		if (preg_match('/(.*?)$/', $s, $matches)) {
+			$x = ((array_key_exists(trim($matches[1]),$item)) ? $item[trim($matches[1])] : EMPTY_STR);
+			if ($x) {
+				return true;
+			}
+			return false;
+		}
+
+		return false;
+	}
+
 }
diff --git a/Zotlabs/Module/Settings/Channel.php b/Zotlabs/Module/Settings/Channel.php
index a0da020b7..840efc162 100644
--- a/Zotlabs/Module/Settings/Channel.php
+++ b/Zotlabs/Module/Settings/Channel.php
@@ -35,6 +35,8 @@ class Channel {
 		$pageflags          = $channel['channel_pageflags'];
 		$existing_adult     = (($pageflags & PAGE_ADULT) ? 1 : 0);
 		$expire             = ((x($_POST, 'expire')) ? intval($_POST['expire']) : 0);
+		$incl               = ((x($_POST['message_filter_incl'])) ? htmlspecialchars_decode(trim($_POST['message_filter_incl']), ENT_QUOTES) : '');
+		$excl               = ((x($_POST['message_filter_excl'])) ? htmlspecialchars_decode(trim($_POST['message_filter_excl']), ENT_QUOTES) : '');
 
 		if ($adult != $existing_adult) {
 			$pageflags = ($pageflags ^ PAGE_ADULT);
@@ -131,6 +133,8 @@ class Channel {
 		set_pconfig(local_channel(), 'system', 'photo_path', $photo_path);
 		set_pconfig(local_channel(), 'system', 'attach_path', $attach_path);
 		set_pconfig(local_channel(), 'system', 'email_notify_host', $mailhost);
+		set_pconfig(local_channel(), 'system', 'message_filter_incl', $incl);
+		set_pconfig(local_channel(), 'system', 'message_filter_excl', $excl);
 
 		$r = q("update channel set channel_pageflags = %d, channel_timezone = '%s',
 				channel_location = '%s', channel_notifyflags = %d, channel_expire_days = %d
@@ -277,6 +281,8 @@ class Channel {
 			'$removeme'                      => t('Remove Channel'),
 			'$removechannel'                 => t('Remove this channel.'),
 			'$expire'                        => ['expire', t('Expire other channel content after this many days'), $expire, t('0 or blank to use the website limit.') . ' ' . ((intval($sys_expire)) ? sprintf(t('This website expires after %d days.'), intval($sys_expire)) : t('This website does not expire imported content.')) . ' ' . t('The website limit takes precedence if lower than your limit.')],
+			'$message_filter_excl'           => ['message_filter_excl', t('Do not import posts with this text'), get_pconfig(local_channel(), 'system', 'message_filter_excl', ''), t('Words one per line or #tags, $categories, /patterns/, lang=xx, lang!=xx - leave blank to import all posts')],
+			'$message_filter_incl'           => ['message_filter_incl', t('Only import posts with this text'), get_pconfig(local_channel(), 'system', 'message_filter_incl', ''), t('Words one per line or #tags, $categories, /patterns/, lang=xx, lang!=xx - leave blank to import all posts')]
 		]);
 
 		call_hooks('settings_form', $o);
diff --git a/Zotlabs/Module/Settings/Privacy.php b/Zotlabs/Module/Settings/Privacy.php
index fbda78a6f..847bb3b8f 100644
--- a/Zotlabs/Module/Settings/Privacy.php
+++ b/Zotlabs/Module/Settings/Privacy.php
@@ -119,7 +119,7 @@ class Privacy {
 			],
 			'$autoperms' => ['autoperms', t('Automatically approve new contacts'), $autoperms, '', [t('No'), t('Yes')]],
 			'$index_opt_out' => ['index_opt_out', t('Opt-out of search engine indexing'), $index_opt_out, '', [t('No'), t('Yes')]],
-			'$group_actor' => ['group_actor', t('Group actor'), $group_actor, t('Allow this channel to act as a forum'), [t('No'), t('Yes')]],
+			'$group_actor' => ['group_actor', t('Group actor'), $group_actor, t('Allow this channel to act as a forum'), [t('No'), t('Yes')]]
 		]);
 
 		return $o;