Merge branch '8.4RC'8.4

9 files changed, 400 insertions, 332 deletions

diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php
index 416aaa0d8..fcf3fb172 100644
--- a/Zotlabs/Lib/Activity.php
+++ b/Zotlabs/Lib/Activity.php
@@ -289,7 +289,7 @@ class Activity {
 
 	static function encode_item_collection($items, $id, $type, $total = 0) {
 
-		if ($total > 30) {
+		if ($total > App::$pager['itemspage']) {
 			$ret = [
 				'id'   => z_root() . '/' . $id,
 				'type' => $type . 'Page',
@@ -591,9 +591,9 @@ class Activity {
 			foreach ($item['term'] as $t) {
 				switch ($t['ttype']) {
 					case TERM_HASHTAG:
-						// id is required so if we don't have a url in the taxonomy, ignore it and keep going.
+						// href is required so if we don't have a url in the taxonomy, ignore it and keep going.
 						if ($t['url']) {
-							$ret[] = ['type' => 'Hashtag', 'id' => $t['url'], 'name' => '#' . $t['term']];
+							$ret[] = ['type' => 'Hashtag', 'href' => $t['url'], 'name' => '#' . $t['term']];
 						}
 						break;
 
@@ -622,6 +622,10 @@ class Activity {
 			$atts = ((is_array($item['attach'])) ? $item['attach'] : json_decode($item['attach'], true));
 			if ($atts) {
 				foreach ($atts as $att) {
+					if (!isset($att['type'], $att['href'])) {
+						continue;
+					}
+
 					if (isset($att['type']) && strpos($att['type'], 'image')) {
 						$ret[] = ['type' => 'Image', 'url' => $att['href']];
 					}
@@ -676,7 +680,7 @@ class Activity {
 
 		$ret = [];
 
-		if (isset($item['attachment'])) {
+		if (isset($item['attachment']) && is_array($item['attachment'])) {
 			$ptr = $item['attachment'];
 			if (!array_key_exists(0, $ptr)) {
 				$ptr = [$ptr];
@@ -1333,7 +1337,8 @@ class Activity {
 			}
 		}
 
-		$x           = PermissionRoles::role_perms('personal');
+		$role = get_pconfig($channel['channel_id'], 'system', 'permissions_role', 'personal');
+		$x = PermissionRoles::role_perms($role);
 		$their_perms = Permissions::FilledPerms($x['perms_connect']);
 
 		if ($contact && $contact['abook_id']) {
@@ -1358,10 +1363,15 @@ class Activity {
 					return;
 
 				case 'Accept':
+					// They accepted our Follow request.
+					// Set default permissions except for send_stream and post_wall
 
-					// They accepted our Follow request - set default permissions
-
-					set_abconfig($channel['channel_id'], $contact['abook_xchan'], 'system', 'their_perms', $their_perms);
+					foreach ($their_perms as $k => $v) {
+						if(in_array($k, ['send_stream', 'post_wall'])) {
+							$v = 0; // Those will be set once we accept their follow request
+						}
+						set_abconfig($channel['channel_id'], $contact['abook_xchan'], 'their_perms', $k, $v);
+					}
 
 					$abook_instance = $contact['abook_instance'];
 
@@ -2215,18 +2225,14 @@ class Activity {
 	}
 
 	static function decode_note($act) {
-
 		$response_activity = false;
-
 		$s = [];
 
 		// These activities should have been handled separately in the Inbox module and should not be turned into posts
 
 		if (
 			in_array($act->type, ['Follow', 'Accept', 'Reject', 'Create', 'Update']) &&
-			is_array($act->obj) &&
-			array_key_exists('type', $act->obj) &&
-			($act->obj['type'] === 'Follow' || ActivityStreams::is_an_actor($act->obj['type']))
+			($act->objprop('type') === 'Follow' || ActivityStreams::is_an_actor($act->objprop('type')))
 		) {
 			return false;
 		}
@@ -2256,7 +2262,7 @@ class Activity {
 			$content = self::get_content($act->obj);
 		}
 
-		$s['mid'] = ((is_array($act->obj) && isset($act->obj['id'])) ? $act->obj['id'] : $act->obj);
+		$s['mid'] = $act->objprop('id') ?: $act->obj;
 
 		if (!$s['mid']) {
 			return false;
@@ -2264,7 +2270,7 @@ class Activity {
 
 		// Friendica sends the diaspora guid in a nonstandard field via AP
 		// If no uuid is provided we will create an uuid v5 from the mid
-		$s['uuid'] = ((is_array($act->obj) && isset($act->obj['diaspora:guid'])) ? $act->obj['diaspora:guid'] : uuid_from_url($s['mid']));
+		$s['uuid'] = (($act->objprop('diaspora:guid')) ?: uuid_from_url($s['mid']));
 
 		$s['parent_mid'] = $act->parent_id;
 
@@ -2272,24 +2278,24 @@ class Activity {
 			$s['created']   = datetime_convert('UTC', 'UTC', $act->data['published']);
 			$s['commented'] = $s['created'];
 		}
-		elseif (is_array($act->obj) && array_key_exists('published', $act->obj)) {
+		elseif ($act->objprop('published')) {
 			$s['created']   = datetime_convert('UTC', 'UTC', $act->obj['published']);
 			$s['commented'] = $s['created'];
 		}
 		if (array_key_exists('updated', $act->data)) {
 			$s['edited'] = datetime_convert('UTC', 'UTC', $act->data['updated']);
 		}
-		elseif (is_array($act->obj) && array_key_exists('updated', $act->obj)) {
+		elseif ($act->objprop('updated')) {
 			$s['edited'] = datetime_convert('UTC', 'UTC', $act->obj['updated']);
 		}
 		if (array_key_exists('expires', $act->data)) {
 			$s['expires'] = datetime_convert('UTC', 'UTC', $act->data['expires']);
 		}
-		elseif (is_array($act->obj) && array_key_exists('expires', $act->obj)) {
+		elseif ($act->objprop('expires')) {
 			$s['expires'] = datetime_convert('UTC', 'UTC', $act->obj['expires']);
 		}
 
-		if ($act->type === 'Invite' && is_array($act->obj) && array_key_exists('type', $act->obj) && $act->obj['type'] === 'Event') {
+		if ($act->type === 'Invite' && $act->objprop('type') === 'Event') {
 			$s['mid'] = $s['parent_mid'] = $act->id;
 		}
 
@@ -2298,9 +2304,18 @@ class Activity {
 			$response_activity = true;
 
 			$s['mid'] = $act->id;
-			$s['uuid'] = ((is_array($act->data) && isset($act->data['diaspora:guid'])) ? $act->data['diaspora:guid'] : uuid_from_url($s['mid']));
+			$s['uuid'] = ((!empty($act->data['diaspora:guid'])) ? $act->data['diaspora:guid'] : uuid_from_url($s['mid']));
+
+			if ($act->objprop('inReplyTo')) {
+				$s['parent_mid'] = $act->objprop('inReplyTo');
+			}
+
+			$s['thr_parent'] = $act->objprop('id') ?: $act->obj;
 
-			$s['parent_mid'] = ((is_array($act->obj) && isset($act->obj['id'])) ? $act->obj['id'] : $act->obj);
+			if (empty($s['parent_mid']) || empty($s['thr_parent'])) {
+				logger('response activity without parent_mid or thr_parent');
+				return;
+			}
 
 			// over-ride the object timestamp with the activity
 
@@ -2312,7 +2327,7 @@ class Activity {
 				$s['edited'] = datetime_convert('UTC', 'UTC', $act->data['updated']);
 			}
 
-			$obj_actor = ((isset($act->obj['actor'])) ? $act->obj['actor'] : $act->get_actor('attributedTo', $act->obj));
+			$obj_actor = $act->objprop('actor') ?: $act->get_actor('attributedTo', $act->obj);
 
 			if (!isset($obj_actor['id'])) {
 				return false;
@@ -2331,7 +2346,7 @@ class Activity {
 			}
 
 			// handle event RSVPs
-			if (($act->obj['type'] === 'Event') || ($act->obj['type'] === 'Invite' && array_path_exists('object/type', $act->obj) && $act->obj['object']['type'] === 'Event')) {
+			if (($act->objprop('type') === 'Event') || ($act->objprop('type') === 'Invite' && array_path_exists('object/type', $act->obj) && $act->obj['object']['type'] === 'Event')) {
 				if ($act->type === 'Accept') {
 					$content['content'] = sprintf(t('Will attend %s\'s event'), $mention) . EOL . EOL . $content['content'];
 				}
@@ -2366,7 +2381,7 @@ class Activity {
 			$s['item_thread_top'] = 1;
 
 			// it is a parent node - decode the comment policy info if present
-			if (isset($act->obj['commentPolicy'])) {
+			if ($act->objprop('commentPolicy')) {
 				$until = strpos($act->obj['commentPolicy'], 'until=');
 				if ($until !== false) {
 					$s['comments_closed'] = datetime_convert('UTC', 'UTC', substr($act->obj['commentPolicy'], $until + 6));
@@ -2392,7 +2407,7 @@ class Activity {
 		$s['summary'] = self::bb_content($content, 'summary');
 		$s['body']    = ((self::bb_content($content, 'bbcode') && (!$response_activity)) ? self::bb_content($content, 'bbcode') : self::bb_content($content, 'content'));
 
-		if (isset($act->obj['quoteUrl'])) {
+		if ($act->objprop('quoteUrl')) {
 			$quote_bbcode = self::get_quote_bbcode($act->obj['quoteUrl']);
 
 			if ($s['body']) {
@@ -2405,21 +2420,24 @@ class Activity {
 		$s['verb'] = self::activity_decode_mapper($act->type);
 
 		// Mastodon does not provide update timestamps when updating poll tallies which means race conditions may occur here.
-		if ($act->type === 'Update' && $act->obj['type'] === 'Question' && $s['edited'] === $s['created']) {
+		if ($act->type === 'Update' && $act->objprop('type') === 'Question' && $s['edited'] === $s['created']) {
 			$s['edited'] = datetime_convert();
 		}
 
-		if (in_array($act->type, ['Delete', 'Undo', 'Tombstone']) || ($act->type === 'Create' && $act->obj['type'] === 'Tombstone')) {
+		if (in_array($act->type, ['Delete', 'Undo', 'Tombstone']) || ($act->type === 'Create' && (isset($act->obj['type']) && $act->obj['type'] === 'Tombstone'))) {
 			$s['item_deleted'] = 1;
 		}
 
-		$s['obj_type'] = self::activity_obj_decode_mapper($act->obj['type']);
+		if (isset($act->obj['type'])) {
+			$s['obj_type'] = self::activity_obj_decode_mapper($act->obj['type']);
+		}
+
 		if ($s['obj_type'] === ACTIVITY_OBJ_NOTE && $s['mid'] !== $s['parent_mid']) {
 			$s['obj_type'] = ACTIVITY_OBJ_COMMENT;
 		}
 
 		$s['obj'] = $act->obj;
-		if (is_array($s['obj']) && array_path_exists('actor/id', $s['obj'])) {
+		if (array_path_exists('actor/id', $s['obj'])) {
 			$s['obj']['actor'] = $s['obj']['actor']['id'];
 		}
 
@@ -2501,23 +2519,21 @@ class Activity {
 			}
 		}
 
-		if (array_key_exists('type', $act->obj)) {
 
-			// Objects that might have media attachments which aren't already provided in the content element.
-			// We'll check specific media objects separately.
+		// Objects that might have media attachments which aren't already provided in the content element.
+		// We'll check specific media objects separately.
 
-			if (in_array($act->obj['type'], ['Article', 'Document', 'Event', 'Note', 'Page', 'Place', 'Question']) && isset($s['attach']) && $s['attach']) {
-				$s = self::bb_attach($s);
-			}
+		if (in_array($act->objprop('type',''), ['Article', 'Document', 'Event', 'Note', 'Page', 'Place', 'Question']) && !empty($s['attach'])) {
+			$s = self::bb_attach($s);
+		}
 
-			if ($act->obj['type'] === 'Question' && in_array($act->type, ['Create', 'Update'])) {
-				if (array_key_exists('endTime', $act->obj)) {
-					$s['comments_closed'] = datetime_convert('UTC', 'UTC', $act->obj['endTime']);
-				}
+		if ($act->objprop('type') === 'Question' && in_array($act->type, ['Create', 'Update'])) {
+			if ($act->objprop('endTime')) {
+				$s['comments_closed'] = datetime_convert('UTC', 'UTC', $act->obj['endTime']);
 			}
 		}
 
-		if (array_key_exists('closed', $act->obj)) {
+		if ($act->objprop('closed')) {
 			$s['comments_closed'] = datetime_convert('UTC', 'UTC', $act->obj['closed']);
 		}
 
@@ -2536,7 +2552,7 @@ class Activity {
 			// right now just link to the largest mp4 we find that will fit in our
 			// standard content region
 
-			if ($act->obj['type'] === 'Video') {
+			if ($act->objprop('type') === 'Video') {
 
 				$vtypes = [
 					'video/mp4',
@@ -2550,7 +2566,7 @@ class Activity {
 
 				// try to find a poster to display on the video element
 
-				if (array_key_exists('icon',$act->obj)) {
+				if ($act->objprop('icon')) {
 					if (is_array($act->obj['icon'])) {
 						if (array_key_exists(0,$act->obj['icon'])) {
 							$ptr = $act->obj['icon'];
@@ -2571,7 +2587,7 @@ class Activity {
 				$tag = (($poster) ? '[video poster="' . $poster . '"]' : '[video]' );
 				$ptr = null;
 
-				if (array_key_exists('url',$act->obj)) {
+				if ($act->objprop('url')) {
 					if (is_array($act->obj['url'])) {
 						if (array_key_exists(0,$act->obj['url'])) {
 							$ptr = $act->obj['url'];
@@ -2617,7 +2633,7 @@ class Activity {
 				}
 			}
 
-			if ($act->obj['type'] === 'Audio') {
+			if ($act->objprop('type') === 'Audio') {
 
 				$atypes = [
 					'audio/mpeg',
@@ -2649,7 +2665,7 @@ class Activity {
 
 			}
 
-			if ($act->obj['type'] === 'Image' && strpos($s['body'], 'zrl=') === false) {
+			if ($act->objprop('type') === 'Image' && strpos($s['body'], 'zrl=') === false) {
 
 				$ptr = null;
 
@@ -2678,7 +2694,7 @@ class Activity {
 			}
 
 
-			if ($act->obj['type'] === 'Page' && !$s['body']) {
+			if ($act->objprop('type') === 'Page' && !$s['body']) {
 
 				$ptr  = null;
 				$purl = EMPTY_STR;
@@ -2718,7 +2734,7 @@ class Activity {
 			}
 		}
 
-		if (in_array($act->obj['type'], ['Note', 'Article', 'Page'])) {
+		if (in_array($act->objprop('type', ''), ['Note', 'Article', 'Page'])) {
 			$ptr = null;
 
 			if (array_key_exists('url', $act->obj)) {
@@ -2753,10 +2769,8 @@ class Activity {
 			$s['item_private'] = 0;
 		}
 
-		if (is_array($act->obj)) {
-			if (array_key_exists('directMessage', $act->obj) && intval($act->obj['directMessage'])) {
-				$s['item_private'] = 2;
-			}
+		if ($act->objprop('directMessage')) {
+			$s['item_private'] = 2;
 		}
 
 		$ap_rawmsg = '';
@@ -2839,7 +2853,6 @@ class Activity {
 		];
 
 		call_hooks('decode_note', $hookinfo);
-
 		return $hookinfo['s'];
 
 	}
@@ -2847,6 +2860,7 @@ class Activity {
 	static function store($channel, $observer_hash, $act, $item, $fetch_parents = true, $force = false) {
 		$is_sys_channel = is_sys_channel($channel['channel_id']);
 		$is_child_node  = false;
+		$parent = null;
 
 		// TODO: not implemented
 		// Pleroma scrobbles can be really noisy and contain lots of duplicate activities. Disable them by default.
@@ -2863,25 +2877,26 @@ class Activity {
 		}
 
 		$allowed = false;
-
-		// TODO: not implemented
 		$permit_mentions = intval(PConfig::Get($channel['channel_id'], 'system','permit_all_mentions') && i_am_mentioned($channel, $item));
 
 		if ($is_child_node) {
 
-			$p = q("select * from item where mid = '%s' and uid = %d and item_wall = 1",
+			$parent = q("select * from item where mid = '%s' and uid = %d",
 				dbesc($item['parent_mid']),
 				intval($channel['channel_id'])
 			);
-			if ($p) {
+
+			// TODO: if we do not have a parent stop here and move the fetch to background?
+
+			if ($parent && $parent[0]['item_wall']) {
 				// set the owner to the owner of the parent
-				$item['owner_xchan'] = $p[0]['owner_xchan'];
+				$item['owner_xchan'] = $parent[0]['owner_xchan'];
 
 				// quietly reject group comment boosts by group owner
 				// (usually only sent via ActivityPub so groups will work on microblog platforms)
 				// This catches those activities if they slipped in via a conversation fetch
 
-				if ($p[0]['parent_mid'] !== $item['parent_mid']) {
+				if ($parent[0]['parent_mid'] !== $item['parent_mid']) {
 					if ($item['verb'] === 'Announce' && $item['author_xchan'] === $item['owner_xchan']) {
 						logger('group boost activity by group owner rejected');
 						return;
@@ -2891,7 +2906,7 @@ class Activity {
 				// check permissions against the author, not the sender
 				$allowed = perm_is_allowed($channel['channel_id'], $item['author_xchan'], 'post_comments');
 				if ((!$allowed) && $permit_mentions) {
-					if ($p[0]['owner_xchan'] === $channel['channel_hash']) {
+					if ($parent[0]['owner_xchan'] === $channel['channel_hash']) {
 						$allowed = false;
 					}
 					else {
@@ -2900,7 +2915,7 @@ class Activity {
 				}
 
 				// TODO: not implemented
-				/*if (absolutely_no_comments($p[0])) {
+				/*if (absolutely_no_comments($parent[0])) {
 					$allowed = false;
 				}*/
 
@@ -2922,13 +2937,14 @@ class Activity {
 			else {
 
 				$allowed = true;
+
 				// reject public stream comments that weren't sent by the conversation owner
 				if ($is_sys_channel && $item['owner_xchan'] !== $observer_hash && !$fetch_parents) {
 					$allowed = false;
 				}
 			}
 
-			if ($p && $p[0]['obj_type'] === 'Question') {
+			if ($parent && $parent[0]['obj_type'] === 'Question') {
 				if ($item['obj_type'] === ACTIVITY_OBJ_COMMENT && $item['title'] && (!$item['body'])) {
 					$item['obj_type'] = 'Answer';
 				}
@@ -2941,7 +2957,7 @@ class Activity {
 			if (perm_is_allowed($channel['channel_id'], ((!empty($item['item_fetched'])) ? $item['author_xchan'] : $observer_hash), 'send_stream') || $is_sys_channel) {
 				$allowed = true;
 			}
-			// TODO: not implemented
+
 			if ($permit_mentions) {
 				$allowed = true;
 			}
@@ -3057,34 +3073,25 @@ class Activity {
 			$item['item_verified'] = 1;
 		}
 
-		$parent = null;
-
 		if ($is_child_node) {
-
-			$parent = q("select * from item where mid = '%s' and uid = %d limit 1",
-				dbesc($item['parent_mid']),
-				intval($item['uid'])
-			);
-
 			if (!$parent) {
 				if (!plugin_is_installed('pubcrawl')) {
 					return;
 				}
-				else {
-					$fetch = false;
 
-					// TODO: debug
-					// if (perm_is_allowed($channel['channel_id'],$observer_hash,'send_stream') && (PConfig::Get($channel['channel_id'],'system','hyperdrive',true) || $act->type === 'Announce')) {
-					if (perm_is_allowed($channel['channel_id'], $observer_hash, 'send_stream') || $is_sys_channel) {
-						$fetch = (($fetch_parents) ? self::fetch_and_store_parents($channel, $observer_hash, $item, $force) : false);
-					}
+				$fetch = false;
 
-					if ($fetch) {
-						$parent = q("select * from item where mid = '%s' and uid = %d limit 1",
-							dbesc($item['parent_mid']),
-							intval($item['uid'])
-						);
-					}
+				// TODO: debug
+				// if (perm_is_allowed($channel['channel_id'],$observer_hash,'send_stream') && (PConfig::Get($channel['channel_id'],'system','hyperdrive',true) || $act->type === 'Announce')) {
+				if (perm_is_allowed($channel['channel_id'], $observer_hash, 'send_stream') || $is_sys_channel) {
+					$fetch = (($fetch_parents) ? self::fetch_and_store_parents($channel, $observer_hash, $item, $force) : false);
+				}
+
+				if ($fetch) {
+					$parent = q("select * from item where mid = '%s' and uid = %d",
+						dbesc($item['parent_mid']),
+						intval($item['uid'])
+					);
 				}
 			}
 
@@ -3093,6 +3100,7 @@ class Activity {
 				return;
 			}
 
+/*
 			if ($parent[0]['parent_mid'] !== $item['parent_mid']) {
 				$item['thr_parent'] = $item['parent_mid'];
 			}
@@ -3100,6 +3108,7 @@ class Activity {
 				$item['thr_parent'] = $parent[0]['parent_mid'];
 			}
 			$item['parent_mid'] = $parent[0]['parent_mid'];
+*/
 
 			/*
 			 *
@@ -3988,18 +3997,18 @@ class Activity {
 
 		switch ($options) {
 			case 'activitypub':
-				$hublocs = q("select * from hubloc left join xchan on hubloc_hash = xchan_hash where hubloc_hash = '%s' and hubloc_deleted = 0 ",
+				$hublocs = q("select * from hubloc left join xchan on hubloc_hash = xchan_hash where hubloc_hash = '%s' and hubloc_deleted = 0 order by hubloc_id desc",
 					dbesc($url)
 				);
 				break;
 			case 'zot6':
-				$hublocs = q("select * from hubloc left join xchan on hubloc_hash = xchan_hash where hubloc_id_url = '%s' and hubloc_deleted = 0 ",
+				$hublocs = q("select * from hubloc left join xchan on hubloc_hash = xchan_hash where hubloc_id_url = '%s' and hubloc_deleted = 0 order by hubloc_id desc",
 					dbesc($url)
 				);
 				break;
 			case 'all':
 			default:
-				$hublocs = q("select * from hubloc left join xchan on hubloc_hash = xchan_hash where ( hubloc_id_url = '%s' OR hubloc_hash = '%s' ) and hubloc_deleted = 0 ",
+				$hublocs = q("select * from hubloc left join xchan on hubloc_hash = xchan_hash where ( hubloc_id_url = '%s' OR hubloc_hash = '%s' ) and hubloc_deleted = 0 order by hubloc_id desc",
 					dbesc($url),
 					dbesc($url)
 				);
@@ -4059,9 +4068,9 @@ class Activity {
 			if ($act->is_valid()) {
 				$content = self::get_content($act->obj);
 
-				$ret .= "[share author='" . urlencode($act->actor['name']) .
+				$ret .= "[share author='" . urlencode($act->actor['name'] ?? $act->actor['preferredUsername']) .
 					"' profile='" . $act->actor['id'] .
-					"' avatar='" . $act->actor['icon']['url'] .
+					"' avatar='" . ($act->actor['icon']['url'] ?? z_root() . '/' . get_default_profile_photo(80)) .
 					"' link='" . $act->obj['id'] .
 					"' auth='" . ((is_matrix_url($act->actor['id'])) ? 'true' : 'false') .
 					"' posted='" . $act->obj['published'] .
diff --git a/Zotlabs/Lib/ActivityStreams.php b/Zotlabs/Lib/ActivityStreams.php
index cfed53b3c..f07f99ac3 100644
--- a/Zotlabs/Lib/ActivityStreams.php
+++ b/Zotlabs/Lib/ActivityStreams.php
@@ -126,6 +126,7 @@ class ActivityStreams {
 			if ((!$this->parent_id) && is_array($this->obj) && isset($this->obj['inReplyTo'])) {
 				$this->parent_id = $this->obj['inReplyTo'];
 			}
+
 			if ((!$this->parent_id) && is_array($this->obj) && isset($this->obj['id'])) {
 				$this->parent_id = $this->obj['id'];
 			}
@@ -154,7 +155,7 @@ class ActivityStreams {
 	 * @return mixed
 	 */
 	public function objprop(string $property, mixed $default = false): mixed {
-		$x = $this->get_property_obj($property,$this->obj);
+		$x = $this->get_property_obj($property, $this->obj);
 		return (isset($x)) ? $x : $default;
 	}
 
diff --git a/Zotlabs/Lib/Connect.php b/Zotlabs/Lib/Connect.php
index 802bbe0f5..4de41526b 100644
--- a/Zotlabs/Lib/Connect.php
+++ b/Zotlabs/Lib/Connect.php
@@ -69,7 +69,8 @@ class Connect {
 		$xchan_hash = '';
 		$sql_options = (($protocol) ? " and xchan_network = '" . dbesc($protocol) . "' " : '');
 
-		$r = q("SELECT * FROM xchan LEFT JOIN hubloc ON xchan_hash = hubloc_hash WHERE ( xchan_hash = '%s' or xchan_url = '%s' or xchan_addr = '%s') $sql_options ORDER BY hubloc_id DESC",
+		// We need both, the xchan and the hubloc here hence use JOIN instead of LEFT JOIN
+		$r = q("SELECT * FROM xchan JOIN hubloc ON xchan_hash = hubloc_hash WHERE ( xchan_hash = '%s' or xchan_url = '%s' or xchan_addr = '%s') $sql_options ORDER BY hubloc_id DESC",
 			dbesc($url),
 			dbesc($url),
 			dbesc($url)
diff --git a/Zotlabs/Lib/Libsync.php b/Zotlabs/Lib/Libsync.php
index d52b501e4..0d383c697 100644
--- a/Zotlabs/Lib/Libsync.php
+++ b/Zotlabs/Lib/Libsync.php
@@ -141,7 +141,7 @@ class Libsync {
 
 		$total = count($synchubs);
 		foreach ($synchubs as $hub) {
-			$hash = random_string();
+			$hash = new_uuid();
 			$n    = Libzot::build_packet($channel, 'sync', $env_recips, json_encode($info), 'hz', $hub['hubloc_sitekey'], $hub['site_crypto']);
 			Queue::insert([
 				'hash'       => $hash,
@@ -771,7 +771,12 @@ class Libsync {
 
 	static function sync_locations($sender, $arr) {
 
-		$ret = [];
+		$ret = [
+			'change_message' => '',
+			'changed' => false,
+			'message' => ''
+		];
+
 		$what = '';
 		$changed = false;
 
@@ -786,7 +791,7 @@ class Libsync {
 
 		if (isset($arr['locations']) && $arr['locations']) {
 
-			$xisting = q("select * from hubloc where hubloc_hash = '%s'",
+			$xisting = q("select * from hubloc where hubloc_hash = '%s' and hubloc_deleted = 0",
 				dbesc($sender['hash'])
 			);
 
diff --git a/Zotlabs/Lib/Libzot.php b/Zotlabs/Lib/Libzot.php
index a8334595f..093670338 100644
--- a/Zotlabs/Lib/Libzot.php
+++ b/Zotlabs/Lib/Libzot.php
@@ -328,7 +328,7 @@ class Libzot {
 
 		logger('zot-info: ' . print_r($record, true), LOGGER_DATA, LOG_DEBUG);
 
-		$x = self::import_xchan($record['data'], (($force) ? UPDATE_FLAGS_FORCED : UPDATE_FLAGS_UPDATED));
+		$x = self::import_xchan($record['data']);
 
 		if (!$x['success']) {
 			return false;
@@ -635,23 +635,18 @@ class Libzot {
 	 *   all internal data structures which need to be updated as a result.
 	 *
 	 * @param array $arr => json_decoded discovery packet
-	 * @param int $ud_flags
-	 *    Determines whether to create a directory update record if any changes occur, default is UPDATE_FLAGS_UPDATED
-	 *    $ud_flags = UPDATE_FLAGS_FORCED indicates a forced refresh where we unconditionally create a directory update record
-	 *      this typically occurs once a month for each channel as part of a scheduled ping to notify the directory
-	 *      that the channel still exists
-	 * @param array $ud_arr
-	 *    If set [typically by update_directory_entry()] indicates a specific update table row and more particularly
-	 *    contains a particular address (ud_addr) which needs to be updated in that table.
-	 *
+
 	 * @return array An associative array with:
 	 *   * \e boolean \b success boolean true or false
 	 *   * \e string \b message (optional) error string only if success is false
 	 */
 
-	static function import_xchan($arr, $ud_flags = UPDATE_FLAGS_UPDATED, $ud_arr = null) {
+	static function import_xchan($arr) {
 
-		$ret     = ['success' => false];
+		$ret = [
+			'success' => false,
+			'message' => ''
+		];
 
 		if (!is_array($arr)) {
 			logger('Not an array: ' . print_r($arr, true), LOGGER_DEBUG);
@@ -665,7 +660,7 @@ class Libzot {
 		 */
 		call_hooks('import_xchan', $arr);
 
-		$dirmode = intval(get_config('system', 'directory_mode'));
+		$dirmode = intval(get_config('system', 'directory_mode', DIRECTORY_MODE_NORMAL));
 
 		$changed = false;
 		$what    = '';
@@ -683,15 +678,11 @@ class Libzot {
 		$sig_methods = ((array_key_exists('signing', $arr) && is_array($arr['signing'])) ? $arr['signing'] : ['sha256']);
 		$verified    = false;
 
-		if (!self::verify($arr['id'], $arr['id_sig'], $arr['public_key'])) {
-			logger('Unable to verify channel signature for ' . $arr['primary_location']['address']);
-			return $ret;
-		}
-		else {
+		if (self::verify($arr['id'], $arr['id_sig'], $arr['public_key'])) {
 			$verified = true;
 		}
-
-		if (!$verified) {
+		else {
+			logger('Unable to verify channel signature for ' . $xchan_hash . ' (' . $arr['primary_location']['address']) . ')';
 			$ret['message'] = t('Unable to verify channel signature');
 			return $ret;
 		}
@@ -714,6 +705,7 @@ class Libzot {
 			if ($arr['photo'] && array_key_exists('updated', $arr['photo']) && $arr['photo']['updated'] > $r[0]['xchan_photo_date'])
 				$import_photos = true;
 
+
 			// if we import an entry from a site that's not ours and either or both of us is off the grid - hide the entry.
 			/** @TODO: check if we're the same directory realm, which would mean we are allowed to see it */
 
@@ -730,7 +722,7 @@ class Libzot {
 				$hidden_changed = 1;
 			if (isset($arr['adult_content']) && intval($r[0]['xchan_selfcensored']) != intval($arr['adult_content']))
 				$adult_changed = 1;
-			if (isset($arr['xchan_deleted']) && intval($r[0]['xchan_deleted']) != intval($arr['deleted']))
+			if (isset($arr['deleted']) && intval($r[0]['xchan_deleted']) != intval($arr['deleted']))
 				$deleted_changed = 1;
 
 			// new style 6-MAR-2019
@@ -920,30 +912,26 @@ class Libzot {
 		$s = Libsync::sync_locations($arr, $arr);
 
 		if ($s) {
-			if (isset($s['change_message']))
+			if (!empty($s['change_message']))
 				$what .= $s['change_message'];
-			if (isset($s['changed']))
-				$changed = $s['changed'];
-			if (isset($s['message']))
+			if (!empty($s['changed']))
+				$changed .= $s['changed'];
+			if (!empty($s['message']))
 				$ret['message'] .= $s['message'];
 		}
 
-		// Which entries in the update table are we interested in updating?
-
-		$address = (($ud_arr && $ud_arr['ud_addr']) ? $ud_arr['ud_addr'] : $arr['primary_location']['address']);
-
-
-		// Are we a directory server of some kind?
 
 		$other_realm = false;
 		$realm       = get_directory_realm();
-		if (array_key_exists('site', $arr)
-			&& array_key_exists('realm', $arr['site'])
-			&& (strpos($arr['site']['realm'], $realm) === false))
+		if (array_key_exists('site', $arr) && array_key_exists('realm', $arr['site']) && (strpos($arr['site']['realm'], $realm) === false)) {
 			$other_realm = true;
+		}
 
+		$address = $arr['primary_location']['url'];
 
-		if ($dirmode != DIRECTORY_MODE_NORMAL) {
+		// Are we a directory server of some kind?
+
+		if ($dirmode !== DIRECTORY_MODE_NORMAL) {
 
 			// We're some kind of directory server. However we can only add directory information
 			// if the entry is in the same realm (or is a sub-realm). Sub-realms are denoted by
@@ -951,7 +939,7 @@ class Libzot {
 			// be in directories for the local realm (foo) and also the RED_GLOBAL realm.
 
 			if (array_key_exists('profile', $arr) && is_array($arr['profile']) && (!$other_realm)) {
-				$profile_changed = Libzotdir::import_directory_profile($xchan_hash, $arr['profile'], $address, $ud_flags, 1);
+				$profile_changed = Libzotdir::import_directory_profile($xchan_hash, $arr['profile']);
 				if ($profile_changed) {
 					$what    .= 'profile ';
 					$changed = true;
@@ -977,21 +965,10 @@ class Libzot {
 			}
 		}
 
-		if (($changed) || ($ud_flags == UPDATE_FLAGS_FORCED)) {
-			$guid = random_string() . '@' . \App::get_hostname();
-			Libzotdir::update_modtime($xchan_hash, $guid, $address, $ud_flags);
-			logger('Changed: ' . $what, LOGGER_DEBUG);
-		}
-		elseif (!$ud_flags) {
-			// nothing changed but we still need to update the updates record
-			q("update updates set ud_flags = ( ud_flags | %d ) where ud_addr = '%s' and (ud_flags & %d) = 0 ",
-				intval(UPDATE_FLAGS_UPDATED),
-				dbesc($address),
-				intval(UPDATE_FLAGS_UPDATED)
-			);
-		}
+		// update updates if anything changed bump the ud_date
+		Libzotdir::update($xchan_hash, $address, $changed);
 
-		if (!x($ret, 'message')) {
+		if (empty($ret['message'])) {
 			$ret['success'] = true;
 			$ret['hash']    = $xchan_hash;
 		}
@@ -1398,7 +1375,7 @@ class Libzot {
 				$uids = ids_to_querystr($uids, 'uid');
 
 				$z = q("SELECT channel_hash FROM channel WHERE channel_hash != '%s' AND channel_id IN ($uids)",
-					dbesc($msg['sender'])
+					dbesc($env['sender'])
 				);
 
 				if ($z) {
@@ -1746,7 +1723,7 @@ class Libzot {
 					// this is just an exercise in futility.
 
 					if (perm_is_allowed($channel['channel_id'], $sender, 'send_stream')) {
-						self::fetch_conversation($channel, $arr['parent_mid']);
+						Master::Summon(['Zotconvo', $channel['channel_id'], $arr['parent_mid']]);
 					}
 
 					continue;
@@ -2037,18 +2014,12 @@ class Libzot {
 				continue;
 			}
 
-			$r = q("select hubloc_hash, hubloc_network from hubloc where hubloc_id_url = '%s' order by hubloc_id desc",
-				dbesc($AS->actor['id'])
-			);
-
+			$r = Activity::get_actor_hublocs($AS->actor['id']);
 			$r = self::zot_record_preferred($r);
-
 			if (!$r) {
 				$y = import_author_xchan(['url' => $AS->actor['id']]);
 				if ($y) {
-					$r = q("select hubloc_hash, hubloc_network from hubloc where hubloc_id_url = '%s'",
-						dbesc($AS->actor['id'])
-					);
+					$r = Activity::get_actor_hublocs($AS->actor['id']);
 					$r = self::zot_record_preferred($r);
 				}
 				if (!$r) {
@@ -2058,22 +2029,29 @@ class Libzot {
 			}
 
 			if (isset($AS->obj['actor']['id']) && $AS->obj['actor']['id'] !== $AS->actor['id']) {
-				$y = import_author_xchan(['url' => $AS->obj['actor']['id']]);
-				if (!$y) {
-					logger('FOF Activity: no object actor');
-					continue;
+				$ro = Activity::get_actor_hublocs($AS->obj['actor']['id']);
+				$ro = self::zot_record_preferred($ro);
+				if (!$ro) {
+					$y = import_author_xchan(['url' => $AS->obj['actor']['id']]);
+					if ($y) {
+						$ro = Activity::get_actor_hublocs($AS->obj['actor']['id']);
+						$ro = self::zot_record_preferred($ro);
+					}
+					if (!$ro) {
+						logger('FOF Activity: no obj actor');
+						continue;
+					}
 				}
 			}
 
 			$arr = Activity::decode_note($AS);
 
 			if (!$arr) {
+				logger('FOF Activity: could not decode');
 				continue;
 			}
 
-			if ($r) {
-				$arr['author_xchan'] = $r['hubloc_hash'];
-			}
+			$arr['author_xchan'] = $r['hubloc_hash'];
 
 			if ($signer) {
 				$arr['owner_xchan'] = $signer[0]['hubloc_hash'];
@@ -2377,19 +2355,6 @@ class Libzot {
 					'deleted'  => (intval($hub['hubloc_deleted']) ? true : false)
 				];
 
-				// version compatibility tweaks
-
-				if (!strpos($z['url_sig'], '.')) {
-					$z['url_sig'] = 'sha256.' . $z['url_sig'];
-				}
-
-				if (!$z['id_url']) {
-					$z['id_url'] = $z['url'] . '/channel/' . substr($z['address'], 0, strpos($z['address'], '@'));
-				}
-				if (!$z['site_id']) {
-					$z['site_id'] = Libzot::make_xchan_hash($z['url'], $z['sitekey']);
-				}
-
 				$ret[] = $z;
 			}
 		}
@@ -2761,12 +2726,12 @@ class Libzot {
 		$e = $r[0];
 		$id = $e['channel_id'];
 
-		$sys_channel     = (intval($e['channel_system']) ? true : false);
+		$sys_channel     = ((empty($e['channel_system'])) ? false : true);
 		$special_channel = (($e['channel_pageflags'] & PAGE_PREMIUM) ? true : false);
 		$adult_channel   = (($e['channel_pageflags'] & PAGE_ADULT) ? true : false);
 		$censored        = (($e['channel_pageflags'] & PAGE_CENSORED) ? true : false);
 		$searchable      = (($e['channel_pageflags'] & PAGE_HIDDEN) ? false : true);
-		$deleted         = (intval($e['xchan_deleted']) ? true : false);
+		$deleted         = ((empty($e['xchan_deleted'])) ? false : true);
 
 		if ($deleted || $censored || $sys_channel)
 			$searchable = false;
@@ -2818,8 +2783,8 @@ class Libzot {
 
 		// Communication details
 
-		$ret['id']     = $e['xchan_guid'];
-		$ret['id_sig'] = self::sign($e['xchan_guid'], $e['channel_prvkey']);
+		$ret['id']     = $e['channel_guid'];
+		$ret['id_sig'] = self::sign($e['channel_guid'], $e['channel_prvkey']);
 
 		$ret['primary_location'] = [
 			'address'         => $e['xchan_addr'],
@@ -2828,10 +2793,10 @@ class Libzot {
 			'follow_url'      => $e['xchan_follow'],
 		];
 
-		$ret['public_key']   = $e['xchan_pubkey'];
+		$ret['public_key']   = $e['channel_pubkey'];
 		$ret['signing_algorithm'] = 'rsa-sha256';
 		$ret['username']     = $e['channel_address'];
-		$ret['name']         = $e['xchan_name'];
+		$ret['name']         = $e['channel_name'];
 		$ret['name_updated'] = $e['xchan_name_date'];
 		$ret['photo']        = [
 			'url'     => $e['xchan_photo_l'],
diff --git a/Zotlabs/Lib/Libzotdir.php b/Zotlabs/Lib/Libzotdir.php
index dfedd0bc8..58138850c 100644
--- a/Zotlabs/Lib/Libzotdir.php
+++ b/Zotlabs/Lib/Libzotdir.php
@@ -172,13 +172,12 @@ class Libzotdir {
 	}
 
 	/**
-	 * @brief Checks the directory mode of this hub.
+	 * @brief fetches updates from known directories
 	 *
 	 * Checks the directory mode of this hub to see if it is some form of directory server. If it is,
 	 * get the directory realm of this hub. Fetch a list of all other directory servers in this realm and request
-	 * a directory sync packet. This will contain both directory updates and new ratings. Store these all in the DB.
-	 * In the case of updates, we will query each of them asynchronously from a poller task. Ratings are stored
-	 * directly if the rater's signature matches.
+	 * a directory sync packet. Store these all in the DB.
+	 * In the case of updates, we will query each of them asynchronously from a poller task.
 	 *
 	 * @param int $dirmode;
 	 */
@@ -233,6 +232,8 @@ class Libzotdir {
 		if (! $r)
 			return;
 
+		$dir_trusted_hosts = array_merge(get_directory_fallback_servers(), get_config('system', 'trusted_directory_servers', []));
+
 		foreach ($r as $rr) {
 			if (! $rr['site_directory'])
 				continue;
@@ -264,31 +265,65 @@ class Libzotdir {
 
 			if (is_array($j['transactions']) && count($j['transactions'])) {
 				foreach ($j['transactions'] as $t) {
-
-					if (empty($t['hash']) || empty($t['transaction_id']) || empty($t['address'])) {
+					if (empty($t['hash']) || empty($t['host']) || empty($t['address'])) {
 						continue;
 					}
 
-					$r = q("select * from updates where ud_guid = '%s' limit 1",
-						dbesc($t['transaction_id'])
+					$r = q("select * from updates where ud_hash = '%s' limit 1",
+						dbesc($t['hash'])
 					);
-					if($r)
-						continue;
 
-					$ud_flags = 0;
-					if (is_array($t['flags']) && in_array('deleted',$t['flags']))
-						$ud_flags |= UPDATE_FLAGS_DELETED;
-					if (is_array($t['flags']) && in_array('forced',$t['flags']))
-						$ud_flags |= UPDATE_FLAGS_FORCED;
-
-					$z = q("insert into updates ( ud_hash, ud_guid, ud_date, ud_flags, ud_addr )
-						values ( '%s', '%s', '%s', %d, '%s' ) ",
-						dbesc($t['hash']),
-						dbesc($t['transaction_id']),
-						dbesc($t['timestamp']),
-						intval($ud_flags),
-						dbesc($t['address'])
-					);
+					if ($r) {
+						$update = 0;
+
+						// no need to look at updates that originated from our own site
+						if ($t['host'] === z_root()) {
+							continue;
+						}
+
+						// there is more recent xchan information
+						if ($r[0]['ud_date'] <= $t['timestamp']) {
+							$update = 1;
+						}
+
+						// the host is trusted and flags have changed - update flags immediately
+						if (in_array($t['host'], $dir_trusted_hosts) &&
+							$rr['site_url'] === $t['host'] &&
+							intval($r[0]['ud_flags']) !== intval($t['flags'])) {
+
+							q("UPDATE updates SET ud_update = %d, ud_flags = %d WHERE ud_id = %d",
+								intval($update),
+								intval($t['flags']),
+								dbesc($r[0]['ud_id'])
+							);
+
+							q("UPDATE xchan SET xchan_censored = %d WHERE xchan_hash = '%s'",
+								intval($t['flags']),
+								dbesc($r[0]['ud_hash'])
+							);
+
+							continue;
+						}
+
+						if (!$update) {
+							continue;
+						}
+
+						q("UPDATE updates SET ud_update = %d WHERE ud_id = %d",
+							intval($update),
+							dbesc($r[0]['ud_id'])
+						);
+					}
+					else {
+						q("insert into updates ( ud_hash, ud_host, ud_date, ud_addr, ud_update, ud_flags )
+							values ( '%s', '%s', '%s', '%s', 1, %d) ",
+							dbesc($t['hash']),
+							dbesc($t['host']),
+							dbesc($t['timestamp']),
+							dbesc($t['address']),
+							dbesc(in_array($t['host'], $dir_trusted_hosts) ? $t['flags'] : 0)
+						);
+					}
 				}
 			}
 		}
@@ -303,8 +338,9 @@ class Libzotdir {
 	 *
 	 * Ignore updating records marked as deleted.
 	 *
-	 * If successful, sets ud_last in the DB to the current datetime for this
+	 * If successful, sets ud_updated in the DB to the current datetime for this
 	 * reddress/webbie.
+	 * Else update ud_last so we can stop trying after 7 days (Daemon/Poller.php)
 	 *
 	 * @param array $ud Entry from update table
 	 */
@@ -313,31 +349,47 @@ class Libzotdir {
 
 		logger('update_directory_entry: ' . print_r($ud,true), LOGGER_DATA);
 
-		if ($ud['ud_addr'] && (! ($ud['ud_flags'] & UPDATE_FLAGS_DELETED))) {
-			$success = false;
-			$zf = [];
+		// TODO: remove this check after all directory servers have version > 8.4
+		// ud_addr will always be the channel url at that time
+		$href = ((strpos($ud['ud_addr'], '://') === false) ? Webfinger::zot_url(punify($ud['ud_addr'])) : punify($ud['ud_addr']));
+		if($href) {
+			$zf = Zotfinger::exec($href);
+			if($zf && array_path_exists('signature/signer',$zf) && $zf['signature']['signer'] === $href && intval($zf['signature']['header_valid'])) {
+				$xc = Libzot::import_xchan($zf['data']);
+
+				// xchan_hash mismatch - this can happen after a site re-install at the same url
+				if ($xc['success'] && $xc['hash'] !== $ud['ud_hash']) {
+					self::delete_by_hash($ud['ud_hash']);
+				}
+
+				// backwards compatibility: Libzot::import_xchan(), where self::update() is called,
+				// will fail with versions < 8.4 if the channel has been locally deleted.
+				// In this case we will update the updates record here without bumping the date
+				// since we could not verify if anything changed.
+				if (!$xc['success'] && !empty($zf['data']['deleted_locally'])) {
+					self::update($ud['ud_hash'], $ud['ud_addr'], false);
+				}
 
-			$href = Webfinger::zot_url(punify($ud['ud_addr']));
-			if($href) {
-				$zf = Zotfinger::exec($href);
-			}
-			if(array_path_exists('signature/signer',$zf) && $zf['signature']['signer'] === $href && intval($zf['signature']['header_valid'])) {
-				$xc = Libzot::import_xchan($zf['data'], 0, $ud);
 				// This is a workaround for a missing xchan_updated column
 				// TODO: implement xchan_updated in the xchan table and update this column instead
-				if($zf['data']['primary_location']['address'] && $zf['data']['primary_location']['url']) {
+				if(!empty($zf['data']['primary_location']['url'])) {
 					q("UPDATE hubloc SET hubloc_updated = '%s' WHERE hubloc_id_url = '%s' AND hubloc_primary = 1",
 						dbesc(datetime_convert()),
 						dbesc($zf['data']['primary_location']['url'])
 					);
 				}
 
-				q("update updates set ud_last = '%s' where ud_addr = '%s'",
-					dbesc(datetime_convert()),
-					dbesc($ud['ud_addr'])
-				);
+				return true;
 			}
 		}
+
+		q("UPDATE updates SET ud_addr = '%s', ud_last = '%s' WHERE ud_hash = '%s'",
+			dbesc($href ? $href : $ud['ud_addr']),
+			dbesc(datetime_convert()),
+			dbesc($ud['ud_hash'])
+		);
+
+		return false;
 	}
 
 
@@ -352,85 +404,78 @@ class Libzotdir {
 	 */
 
 	static function local_dir_update($uid, $force) {
+		logger('local_dir_update uid: ' . $uid, LOGGER_DEBUG);
 
-
-		logger('local_dir_update: uid: ' . $uid, LOGGER_DEBUG);
-
-		$p = q("select channel.channel_hash, channel_address, channel_timezone, profile.* from profile left join channel on channel_id = uid where uid = %d and is_default = 1",
+		$p = q("select channel.channel_hash, channel_address, channel_timezone, profile.*, xchan.xchan_hidden, xchan.xchan_url from profile left join channel on channel_id = uid left join xchan on channel_hash = xchan_hash where profile.uid = %d and profile.is_default = 1",
 			intval($uid)
 		);
 
-		$profile = array();
+		if (!$p) {
+			logger('profile not found');
+			return;
+		}
+
+		$profile = [];
 		$profile['encoding'] = 'zot';
 
-		if ($p) {
-			$hash = $p[0]['channel_hash'];
-
-			$profile['description'] = $p[0]['pdesc'];
-			$profile['birthday']    = $p[0]['dob'];
-			if ($age = age($p[0]['dob'],$p[0]['channel_timezone'],''))
-				$profile['age'] = $age;
-
-			$profile['gender']      = $p[0]['gender'];
-			$profile['marital']     = $p[0]['marital'];
-			$profile['sexual']      = $p[0]['sexual'];
-			$profile['locale']      = $p[0]['locality'];
-			$profile['region']      = $p[0]['region'];
-			$profile['postcode']    = $p[0]['postal_code'];
-			$profile['country']     = $p[0]['country_name'];
-			$profile['about']       = $p[0]['about'];
-			$profile['homepage']    = $p[0]['homepage'];
-			$profile['hometown']    = $p[0]['hometown'];
-
-			if ($p[0]['keywords']) {
-				$tags = array();
-				$k = explode(' ', $p[0]['keywords']);
-				if ($k)
-					foreach ($k as $kk)
-						if (trim($kk))
-							$tags[] = trim($kk);
-
-				if ($tags)
-					$profile['keywords'] = $tags;
-			}
+		$hash = $p[0]['channel_hash'];
+
+		$profile['description'] = $p[0]['pdesc'];
+		$profile['birthday']    = $p[0]['dob'];
+		if ($age = age($p[0]['dob'],$p[0]['channel_timezone'],''))
+			$profile['age'] = $age;
+
+		$profile['gender']      = $p[0]['gender'];
+		$profile['marital']     = $p[0]['marital'];
+		$profile['sexual']      = $p[0]['sexual'];
+		$profile['locale']      = $p[0]['locality'];
+		$profile['region']      = $p[0]['region'];
+		$profile['postcode']    = $p[0]['postal_code'];
+		$profile['country']     = $p[0]['country_name'];
+		$profile['about']       = $p[0]['about'];
+		$profile['homepage']    = $p[0]['homepage'];
+		$profile['hometown']    = $p[0]['hometown'];
+
+		if ($p[0]['keywords']) {
+			$tags = array();
+			$k = explode(' ', $p[0]['keywords']);
+			if ($k)
+				foreach ($k as $kk)
+					if (trim($kk))
+						$tags[] = trim($kk);
+
+			if ($tags)
+				$profile['keywords'] = $tags;
+		}
 
-			$hidden = (1 - intval($p[0]['publish']));
+		$hidden = (1 - intval($p[0]['publish']));
 
-			logger('hidden: ' . $hidden);
+		logger('hidden: ' . $hidden);
 
-			$r = q("select xchan_hidden from xchan where xchan_hash = '%s'",
-				dbesc($p[0]['channel_hash'])
+		if(intval($p[0]['xchan_hidden']) !== $hidden) {
+			q("update xchan set xchan_hidden = %d where xchan_hash = '%s'",
+				intval($hidden),
+				dbesc($hash)
 			);
+		}
 
-			if(intval($r[0]['xchan_hidden']) != $hidden) {
-				$r = q("update xchan set xchan_hidden = %d where xchan_hash = '%s'",
-					intval($hidden),
-					dbesc($hash)
-				);
-			}
-
-			$arr = [ 'channel_id' => $uid, 'hash' => $hash, 'profile' => $profile ];
-			call_hooks('local_dir_update', $arr);
-
-			$address = channel_reddress($p[0]);
-
-			if (perm_is_allowed($uid, '', 'view_profile')) {
-				self::import_directory_profile($hash, $arr['profile'], $address, 0);
-			}
-			else {
-				// they may have made it private
-				q("delete from xprof where xprof_hash = '%s'",
-					dbesc($hash)
-				);
-				q("delete from xtag where xtag_hash = '%s'",
-					dbesc($hash)
-				);
-			}
+		$arr = [ 'channel_id' => $uid, 'hash' => $hash, 'profile' => $profile ];
+		call_hooks('local_dir_update', $arr);
 
+		if (perm_is_allowed($uid, '', 'view_profile')) {
+			self::import_directory_profile($hash, $arr['profile']);
+		}
+		else {
+			// they may have made it private
+			q("delete from xprof where xprof_hash = '%s'",
+				dbesc($hash)
+			);
+			q("delete from xtag where xtag_hash = '%s'",
+				dbesc($hash)
+			);
 		}
 
-		$ud_hash = random_string() . '@' . \App::get_hostname();
-		self::update_modtime($hash, $ud_hash, channel_reddress($p[0]),(($force) ? UPDATE_FLAGS_FORCED : UPDATE_FLAGS_UPDATED));
+		self::update($hash, $p[0]['xchan_url']);
 	}
 
 
@@ -440,13 +485,10 @@ class Libzotdir {
 	 *
 	 * @param string $hash
 	 * @param array $profile
-	 * @param string $addr
-	 * @param number $ud_flags (optional) UPDATE_FLAGS_UPDATED
-	 * @param number $suppress_update (optional) default 0
 	 * @return boolean $updated if something changed
 	 */
 
-	static function import_directory_profile($hash, $profile, $addr, $ud_flags = UPDATE_FLAGS_UPDATED, $suppress_update = 0) {
+	static function import_directory_profile($hash, $profile) {
 
 		logger('import_directory_profile', LOGGER_DEBUG);
 		if (! $hash)
@@ -479,7 +521,12 @@ class Libzotdir {
 		$clean = array();
 		if (array_key_exists('keywords', $profile) and is_array($profile['keywords'])) {
 			self::import_directory_keywords($hash,$profile['keywords']);
+
 			foreach ($profile['keywords'] as $kw) {
+				if (in_array($kw, $clean)) {
+					continue;
+				}
+
 				$kw = trim(htmlspecialchars($kw,ENT_COMPAT, 'UTF-8', false));
 				$kw = trim($kw, ',');
 				$clean[] = $kw;
@@ -586,9 +633,6 @@ class Libzotdir {
 		 */
 		call_hooks('import_directory_profile', $d);
 
-		if (($d['update']) && (! $suppress_update))
-			self::update_modtime($arr['xprof_hash'],random_string() . '@' . \App::get_hostname(), $addr, $ud_flags);
-
 		return $d['update'];
 	}
 
@@ -606,6 +650,10 @@ class Libzotdir {
 			dbesc($hash)
 		);
 
+		$xchan = q("select xchan_censored from xchan where xchan_hash = '%s'",
+			dbesc($hash)
+		);
+
 		if($r) {
 			foreach($r as $rr)
 				$existing[] = $rr['xtag_term'];
@@ -613,6 +661,10 @@ class Libzotdir {
 
 		$clean = array();
 		foreach($keywords as $kw) {
+			if (in_array($kw, $clean)) {
+				continue;
+			}
+
 			$kw = trim(htmlspecialchars($kw,ENT_COMPAT, 'UTF-8', false));
 			$kw = trim($kw, ',');
 			$clean[] = $kw;
@@ -627,9 +679,10 @@ class Libzotdir {
 		}
 		foreach($clean as $x) {
 			if(! in_array($x, $existing)) {
-				$r = q("insert into xtag ( xtag_hash, xtag_term, xtag_flags) values ( '%s' ,'%s', 0 )",
+				$r = q("insert into xtag ( xtag_hash, xtag_term, xtag_flags) values ( '%s' ,'%s', %d )",
 					dbesc($hash),
-					dbesc($x)
+					dbesc($x),
+					intval($xchan[0]['xchan_censored'])
 				);
 			}
 		}
@@ -639,13 +692,12 @@ class Libzotdir {
 	/**
 	 * @brief
 	 *
-	 * @param string $hash
-	 * @param string $guid
-	 * @param string $addr
-	 * @param int $flags (optional) default 0
+	 * @param string $hash the channel hash
+	 * @param string $addr the channel url
+	 * @param bool $bump_date (optional) default true
 	 */
 
-	static function update_modtime($hash, $guid, $addr, $flags = 0) {
+	static function update($hash, $addr, $bump_date = true, $flag = null) {
 
 		$dirmode = intval(get_config('system', 'directory_mode'));
 
@@ -653,26 +705,70 @@ class Libzotdir {
 			return;
 		}
 
-		if (empty($hash) || empty($guid) || empty($addr)) {
+		if (empty($hash) || empty($addr)) {
 			return;
 		}
 
-		if($flags) {
-			q("insert into updates (ud_hash, ud_guid, ud_date, ud_flags, ud_addr ) values ( '%s', '%s', '%s', %d, '%s' )",
-				dbesc($hash),
-				dbesc($guid),
-				dbesc(datetime_convert()),
-				intval($flags),
-				dbesc($addr)
-			);
+		$u = q("SELECT * FROM updates WHERE ud_hash = '%s' LIMIT 1",
+			dbesc($hash)
+		);
+
+		$date_sql = '';
+		if ($bump_date) {
+			$date_sql = "ud_date = '" . dbesc(datetime_convert()) . "',";
 		}
-		else {
-			q("update updates set ud_flags = ( ud_flags | %d ) where ud_addr = '%s' and not (ud_flags & %d)>0 ",
-				intval(UPDATE_FLAGS_UPDATED),
+
+		$flag_sql = '';
+		if ($flag !== null) {
+			$flag_sql = "ud_flags = '" . intval($flag) . "',";
+		}
+
+
+		if ($u) {
+			$x = q("UPDATE updates SET $date_sql $flag_sql ud_last = '%s', ud_host = '%s', ud_addr = '%s', ud_update = 0 WHERE ud_id = %d",
+				dbesc(NULL_DATE),
+				dbesc(z_root()),
 				dbesc($addr),
-				intval(UPDATE_FLAGS_UPDATED)
+				intval($u[0]['ud_id'])
 			);
+
+			return;
+		}
+
+		q("INSERT INTO updates (ud_hash, ud_host, ud_date, ud_addr, ud_flags) VALUES ( '%s', '%s', '%s', '%s', %d )",
+			dbesc($hash),
+			dbesc(z_root()),
+			dbesc(datetime_convert()),
+			dbesc($addr),
+			intval($flag)
+		);
+
+		return;
+
+	}
+
+
+	/**
+	 * @brief deletes a entry in updates by hash
+	 *
+	 * @param string $hash the channel hash
+	 * @return boolean
+	 */
+
+	static function delete_by_hash($hash) {
+		if (!$hash) {
+			return false;
+		}
+
+		$x = q("DELETE FROM updates WHERE ud_hash = '%s'",
+			dbesc($hash)
+		);
+
+		if ($x) {
+			return true;
 		}
+
+		return false;
 	}
 
 }
diff --git a/Zotlabs/Lib/Queue.php b/Zotlabs/Lib/Queue.php
index 348a2a079..942a633ef 100644
--- a/Zotlabs/Lib/Queue.php
+++ b/Zotlabs/Lib/Queue.php
@@ -57,7 +57,6 @@ class Queue {
 			outq_priority = outq_priority + %d,
 			outq_scheduled = '%s'
 			WHERE outq_hash = '%s'",
-
 			dbesc(datetime_convert()),
 			intval($add_priority),
 			dbesc($next),
@@ -85,29 +84,16 @@ class Queue {
 			// entries still exist for it. This fixes an issue where one immediate delivery left everything
 			// else for that site undeliverable since all the other entries had been pushed far into the future.
 
-			$x = null;
-			$sql_quirks = ((get_config('system', 'db_skip_locked_supported')) ? 'SKIP LOCKED' : 'NOWAIT');
-
-			q("START TRANSACTION");
-
-			$r = q("SELECT outq_hash FROM outq WHERE outq_posturl = '%s' LIMIT 1 FOR UPDATE $sql_quirks",
+			$r = q("SELECT outq_hash, outq_posturl FROM outq WHERE outq_posturl = '%s' LIMIT 1",
 				dbesc($record[0]['outq_posturl'])
 			);
 
 			if ($r) {
-				$x = q("UPDATE outq SET outq_scheduled = '%s' WHERE outq_hash = '%s'",
-					dbesc(datetime_convert()),
-					dbesc($r[0]['outq_hash'])
+				$hashes = ids_to_querystr($r, 'outq_hash', true);
+				$x = q("UPDATE outq SET outq_scheduled = '%s' WHERE outq_hash IN ($hashes)",
+					dbesc(datetime_convert())
 				);
 			}
-
-			if ($x) {
-				q("COMMIT");
-			}
-			else {
-				q("ROLLBACK");
-			}
-
 		}
 	}
 
@@ -260,7 +246,7 @@ class Queue {
 
 			if($result['success']) {
 				logger('deliver: remote zot delivery succeeded to ' . $outq['outq_posturl']);
-				Libzot::process_response($outq['outq_posturl'],$result, $outq);
+				Libzot::process_response($outq['outq_posturl'], $result, $outq);
 			}
 			else {
 				logger('deliver: remote zot delivery failed to ' . $outq['outq_posturl']);
diff --git a/Zotlabs/Lib/QueueWorker.php b/Zotlabs/Lib/QueueWorker.php
index a1c13ef8a..1c74b42d8 100644
--- a/Zotlabs/Lib/QueueWorker.php
+++ b/Zotlabs/Lib/QueueWorker.php
@@ -24,7 +24,8 @@ class QueueWorker {
 	// Exceptions for processtimeout ($workermaxage) value.
 	// Currently the value is overriden with 3600 seconds (1h).
 	public static $long_running_cmd = [
-		'Queue'
+		'Queue',
+		'Expire'
 	];
 
 	private static function qstart() {
@@ -147,6 +148,10 @@ class QueueWorker {
 		);
 
 		if ($r) {
+			// TODO: some long running services store their pid in config.procid.daemon
+			// we could possibly check if a pid exist and check if the process is still alive
+			// prior to reseting workerq_reservationid
+
 			$ids = ids_to_querystr($r, 'workerq_id');
 			$u = dbq("update workerq set workerq_reservationid = null where workerq_id in ($ids)");
 		}
diff --git a/Zotlabs/Lib/ThreadItem.php b/Zotlabs/Lib/ThreadItem.php
index d1b386c42..cf877ed92 100644
--- a/Zotlabs/Lib/ThreadItem.php
+++ b/Zotlabs/Lib/ThreadItem.php
@@ -197,7 +197,7 @@ class ThreadItem {
 		$response_verbs = array('like');
 		if(feature_enabled($conv->get_profile_owner(),'dislike'))
 			$response_verbs[] = 'dislike';
-		if($item['obj_type'] === ACTIVITY_OBJ_EVENT) {
+		if(in_array($item['obj_type'], ['Event', ACTIVITY_OBJ_EVENT])) {
 			$response_verbs[] = 'attendyes';
 			$response_verbs[] = 'attendno';
 			$response_verbs[] = 'attendmaybe';
@@ -299,7 +299,7 @@ class ThreadItem {
 		}
 
 		$has_event = false;
-		if(($item['obj_type'] === ACTIVITY_OBJ_EVENT) && $conv->get_profile_owner() == local_channel())
+		if((in_array($item['obj_type'], ['Event', ACTIVITY_OBJ_EVENT])) && $conv->get_profile_owner() == local_channel())
 			$has_event = true;
 
 		$like = [];