Merge branch '5.0RC'5.0

8 files changed, 196 insertions, 58 deletions

diff --git a/Zotlabs/Daemon/Cron.php b/Zotlabs/Daemon/Cron.php
index 8fa31e6ce..703d6ce08 100644
--- a/Zotlabs/Daemon/Cron.php
+++ b/Zotlabs/Daemon/Cron.php
@@ -2,6 +2,8 @@
 
 namespace Zotlabs\Daemon;
 
+use Zotlabs\Lib\Libsync;
+
 class Cron {
 
 	static public function run($argc,$argv) {
@@ -36,10 +38,14 @@ class Cron {
 
 		Master::Summon(array('Poller'));
 
-		// maintenance for mod sharedwithme - check for updated items and remove them
-
-		require_once('include/sharedwithme.php');
-		apply_updates();
+		/**
+		 * Chatpresence: if somebody hasn't pinged recently, they've most likely left the page
+		 * and shouldn't count as online anymore. We allow an expection for bots.
+		 */
+		q("delete from chatpresence where cp_last < %s - INTERVAL %s and cp_client != 'auto' ",
+			db_utcnow(),
+			db_quoteinterval('3 MINUTE')
+		);
 	
 		// expire any expired mail
 
@@ -138,7 +144,7 @@ class Cron {
 					if($z) {
 						xchan_query($z);
 						$sync_item = fetch_post_tags($z);
-						build_sync_packet($sync_item[0]['uid'],
+						Libsync::build_sync_packet($sync_item[0]['uid'],
 							[ 
 								'item' => [ encode_item($sync_item[0],true) ]
 							]
@@ -149,6 +155,11 @@ class Cron {
 			}
 		}
 
+
+		// check if any connections transitioned to zot6 and upgrade the connections to zot6 at this hub if so.
+		require_once('include/connections.php');
+		z6trans_connections();
+
 		require_once('include/attach.php');
 		attach_upgrade();
 
diff --git a/Zotlabs/Daemon/Cron_daily.php b/Zotlabs/Daemon/Cron_daily.php
index eebdb0229..07533cc6e 100644
--- a/Zotlabs/Daemon/Cron_daily.php
+++ b/Zotlabs/Daemon/Cron_daily.php
@@ -44,6 +44,12 @@ class Cron_daily {
 			db_utcnow(), db_quoteinterval('1 YEAR')
 		);
 
+		// expire anonymous sse notification entries once a day
+
+		q("delete from xconfig where xchan like '%s'",
+			dbesc('sse_id.%')
+		);
+
 		// Clean up emdedded content cache
 		q("DELETE FROM cache WHERE updated < %s - INTERVAL %s",
 		    db_utcnow(),
@@ -90,6 +96,7 @@ class Cron_daily {
 		Master::Summon(array('Cli_suggest'));
 
 		remove_obsolete_hublocs();
+
 		z6_discover();
 
 		call_hooks('cron_daily',datetime_convert());
diff --git a/Zotlabs/Daemon/Directory.php b/Zotlabs/Daemon/Directory.php
index c8cdafdf5..c698995c2 100644
--- a/Zotlabs/Daemon/Directory.php
+++ b/Zotlabs/Daemon/Directory.php
@@ -2,10 +2,9 @@
 
 namespace Zotlabs\Daemon;
 
-require_once('include/zot.php');
-require_once('include/dir_fns.php');
-require_once('include/queue_fn.php');
-
+use Zotlabs\Lib\Libzot;
+use Zotlabs\Lib\Libzotdir;
+use Zotlabs\Lib\Queue;
 
 class Directory {
 
@@ -42,7 +41,7 @@ class Directory {
 
 			// this is an in-memory update and we don't need to send a network packet.
 
-			local_dir_update($argv[1],$force);
+			Libzotdir::local_dir_update($argv[1],$force);
 
 			q("update channel set channel_dirdate = '%s' where channel_id = %d",
 				dbesc(datetime_convert()),
@@ -58,13 +57,15 @@ class Directory {
 
 		// otherwise send the changes upstream
 
-		$directory = find_upstream_directory($dirmode);
+		$directory = Libzotdir::find_upstream_directory($dirmode);
 		$url = $directory['url'] . '/post';
 
 		// ensure the upstream directory is updated
 
-		$packet = zot_build_packet($channel,(($force) ? 'force_refresh' : 'refresh'));
-		$z = zot_zot($url,$packet);
+
+		$packet = Libzot::build_packet($channel,(($force) ? 'force_refresh' : 'refresh'));
+		$z = Libzot::zot($url,$packet,$channel);
+
 
 		// re-queue if unsuccessful
 
@@ -76,7 +77,7 @@ class Directory {
 
 			$hash = random_string();
 
-			queue_insert(array(
+			Queue::insert(array(
 				'hash'       => $hash,
 				'account_id' => $channel['channel_account_id'],
 				'channel_id' => $channel['channel_id'],
diff --git a/Zotlabs/Daemon/Expire.php b/Zotlabs/Daemon/Expire.php
index 398425861..a688d6f97 100644
--- a/Zotlabs/Daemon/Expire.php
+++ b/Zotlabs/Daemon/Expire.php
@@ -8,6 +8,15 @@ class Expire {
 	static public function run($argc,$argv){
 
 		cli_startup();
+		
+		$pid = get_config('expire', 'procid', false);
+		if ($pid && (function_exists('posix_kill') ? posix_kill($pid, 0) : true)) {
+		    logger('Expire: procedure already run with pid ' . $pid, LOGGER_DEBUG);
+		    return;
+		}
+		
+		$pid = getmypid();
+		set_config('expire', 'procid', $pid);
 
 		// perform final cleanup on previously delete items
 
@@ -32,7 +41,7 @@ class Expire {
 		if (intval(get_config('system', 'optimize_items')))
 			q("optimize table item");
 
-		logger('expire: start', LOGGER_DEBUG);
+		logger('expire: start with pid ' . $pid, LOGGER_DEBUG);
 
 		$site_expire = intval(get_config('system', 'default_expire_days'));
 		$commented_days = intval(get_config('system','active_expire_days'));
@@ -90,5 +99,7 @@ class Expire {
 
 			logger('Expire: sys: done', LOGGER_DEBUG);
 		}
+		
+		del_config('expire', 'procid');
 	}
 }
diff --git a/Zotlabs/Daemon/Gprobe.php b/Zotlabs/Daemon/Gprobe.php
index f1ffb2d81..6951aa1d4 100644
--- a/Zotlabs/Daemon/Gprobe.php
+++ b/Zotlabs/Daemon/Gprobe.php
@@ -2,7 +2,9 @@
 
 namespace Zotlabs\Daemon;
 
-require_once('include/zot.php');
+use Zotlabs\Lib\Libzot;
+use Zotlabs\Lib\Webfinger;
+use Zotlabs\Lib\Zotfinger;
 
 // performs zot_finger on $argv[1], which is a hex_encoded webbie/reddress
 
@@ -22,12 +24,16 @@ class Gprobe {
 		);
 
 		if(! $r) {
-			$j = \Zotlabs\Zot\Finger::run($url,null);
-			if($j['success']) {
-				$y = import_xchan($j);
+			$href = Webfinger::zot_url(punify($url));
+			if($href) {
+				$zf = Zotfinger::exec($href, null);
+			}
+			if(is_array($zf) && array_path_exists('signature/signer',$zf) && $zf['signature']['signer'] === $href && intval($zf['signature']['header_valid'])) {
+				$xc = Libzot::import_xchan($zf['data']);
 			}
 		}
 
 		return;
+
 	}
 }
diff --git a/Zotlabs/Daemon/Importfile.php b/Zotlabs/Daemon/Importfile.php
index c68ed21cf..749949679 100644
--- a/Zotlabs/Daemon/Importfile.php
+++ b/Zotlabs/Daemon/Importfile.php
@@ -2,6 +2,8 @@
 
 namespace Zotlabs\Daemon;
 
+use Zotlabs\Lib\Libsync;
+
 class Importfile {
 
 	static public function run($argc,$argv){
@@ -40,7 +42,7 @@ class Importfile {
 
 		$sync = attach_export_data($channel,$hash);
 		if($sync)
-			build_sync_packet($channel['channel_id'],array('file' => array($sync)));
+			Libsync::build_sync_packet($channel['channel_id'],array('file' => array($sync)));
 	
 		return;
 	}
diff --git a/Zotlabs/Daemon/Notifier.php b/Zotlabs/Daemon/Notifier.php
index 1d0be10d9..8ea75af61 100644
--- a/Zotlabs/Daemon/Notifier.php
+++ b/Zotlabs/Daemon/Notifier.php
@@ -3,6 +3,7 @@
 namespace Zotlabs\Daemon;
 
 use Zotlabs\Lib\Libzot;
+use Zotlabs\Lib\Activity;
 
 require_once('include/queue_fn.php');
 require_once('include/html2plain.php');
@@ -44,18 +45,18 @@ require_once('include/bbcode.php');
  *
  * where COMMAND is one of the following:
  *
- *		activity				(in diaspora.php, dfrn_confirm.php, profiles.php)
- *		comment-import			(in diaspora.php, items.php)
- *		comment-new				(in item.php)
- *		drop					(in diaspora.php, items.php, photos.php)
- *		edit_post				(in item.php)
- *		event					(in events.php)
- *		expire					(in items.php)
- *		like					(in like.php, poke.php)
- *		mail					(in message.php)
- *		tag						(in photos.php, poke.php, tagger.php)
- *		tgroup					(in items.php)
- *		wall-new				(in photos.php, item.php)
+ *	activity		(in diaspora.php, dfrn_confirm.php, profiles.php)
+ *	comment-import		(in diaspora.php, items.php)
+ *	comment-new		(in item.php)
+ *	drop			(in diaspora.php, items.php, photos.php)
+ *	edit_post		(in item.php)
+ *	event			(in events.php)
+ *	expire			(in items.php)
+ *	like			(in like.php, poke.php)
+ *	mail			(in message.php)
+ *	tag			(in photos.php, poke.php, tagger.php)
+ *	tgroup			(in items.php)
+ *	wall-new		(in photos.php, item.php)
  *
  * and ITEM_ID is the id of the item in the database that needs to be sent to others.
  *
@@ -65,9 +66,10 @@ require_once('include/bbcode.php');
  *       permission_reject      abook_id
  *       permission_update      abook_id
  *       refresh_all            channel_id
+ *       purge                  channel_id            xchan_hash
  *       purge_all              channel_id
  *       expire                 channel_id
- *       relay					item_id (item was relayed to owner, we will deliver it as owner)
+ *       relay			item_id (item was relayed to owner, we will deliver it as owner)
  *       single_activity        item_id (deliver to a singleton network from the appropriate clone)
  *       single_mail            mail_id (deliver to a singleton network from the appropriate clone)
  *       location               channel_id
@@ -239,25 +241,40 @@ class Notifier {
 			$packet_type = 'location';
 			$location = true;
 		}
+		elseif($cmd === 'purge') {
+			$xchan = $argv[3];
+			logger('notifier: purge: ' . $item_id . ' => ' . $xchan);
+			if (! $xchan) {
+				return;
+			}
+
+			$channel     = channelx_by_n($item_id);
+			$recipients[]  = $xchan;
+			$private     = true;
+			$packet_type = 'purge';
+			$packet_recips[] = ['hash' => $xchan];
+		}
 		elseif($cmd === 'purge_all') {
+
 			logger('notifier: purge_all: ' . $item_id);
-			$s = q("select * from channel where channel_id = %d limit 1",
-				intval($item_id)
-			);
-			if($s)
-				$channel = $s[0];
-			$uid = $item_id;
-			$recipients = array();
+			$channel = channelx_by_n($item_id);
+
+			$recipients = [];
 			$r = q("select abook_xchan from abook where abook_channel = %d and abook_self = 0",
 				intval($item_id)
 			);
-			if($r) {
-				foreach($r as $rr) {
-					$recipients[] = $rr['abook_xchan'];
-				}
+			if (! $r) {
+				return;
+			}
+			foreach ($r as $rr) {
+				$recipients[] = $rr['abook_xchan'];
+				$packet_recips[] = ['hash' => $rr['abook_xchan']];
 			}
+
 			$private = false;
 			$packet_type = 'purge';
+
+
 		}
 		else {
 
@@ -277,6 +294,12 @@ class Notifier {
 			$r = fetch_post_tags($r);
 		
 			$target_item = $r[0];
+
+			if(in_array($target_item['author']['xchan_network'], ['rss', 'anon'])) {
+				logger('notifier: target item author is not a fetchable actor', LOGGER_DEBUG);
+				return;
+			}
+
 			$deleted_item = false;
 
 			if(intval($target_item['item_deleted'])) {
@@ -360,15 +383,24 @@ class Notifier {
 
 			$encoded_item = encode_item($target_item);
 
-			// activitystreams version
-			$m = get_iconfig($target_item,'activitystreams','signed_data');
+			// Re-use existing signature unless the activity type changed to a Tombstone, which won't verify.
+			$m = ((intval($target_item['item_deleted'])) ? '' : get_iconfig($target_item,'activitystreams','signed_data'));
+
 			if($m) {
 				$activity = json_decode($m,true);
 			}
 			else {
-				$activity = \Zotlabs\Lib\Activity::encode_activity($target_item);
+				$activity = array_merge(['@context' => [
+					ACTIVITYSTREAMS_JSONLD_REV,
+					'https://w3id.org/security/v1',
+					z_root() . ZOT_APSCHEMA_REV
+					]], Activity::encode_activity($target_item)
+				);
 			}		
 
+			logger('target_item: ' . print_r($target_item,true), LOGGER_DEBUG);
+			logger('encoded: ' . print_r($activity,true), LOGGER_DEBUG);
+
 			// Send comments to the owner to re-deliver to everybody in the conversation
 			// We only do this if the item in question originated on this site. This prevents looping.
 			// To clarify, a site accepting a new comment is responsible for sending it to the owner for relay.
@@ -467,7 +499,6 @@ class Notifier {
 
 		$details = q("select xchan_hash, xchan_network, xchan_addr, xchan_guid, xchan_guid_sig from xchan where xchan_hash in (" . protect_sprintf(implode(',',$recipients)) . ")");
 
-
 		$recip_list = array();
 
 		if($details) {
@@ -532,18 +563,40 @@ class Notifier {
 		// Now we have collected recipients (except for external mentions, FIXME)
 		// Let's reduce this to a set of hubs; checking that the site is not dead.
 
-		$r = q("select hubloc.*, site.site_crypto, site.site_flags from hubloc left join site on site_url = hubloc_url where hubloc_hash in (" . protect_sprintf(implode(',',$recipients)) . ") 
-			and hubloc_error = 0 and hubloc_deleted = 0 and ( site_dead = 0 OR site_dead is null ) "
-		);		
+		$hubs = q("select hubloc.*, site.site_crypto, site.site_flags, site.site_version, site.site_project, site.site_dead from hubloc left join site on site_url = hubloc_url 
+			where hubloc_hash in (" . protect_sprintf(implode(',',$recipients)) . ") 
+			and hubloc_error = 0 and hubloc_deleted = 0"
+		);
+
+		// public posts won't make it to the local public stream unless there's a recipient on this site. 
+		// This code block sees if it's a public post and localhost is missing, and if so adds an entry for the local sys channel to the $hubs list
+
+		if (! $private) {
+			$found_localhost = false;
+			if ($hubs) {
+				foreach ($hubs as $h) {
+					if ($h['hubloc_url'] === z_root()) {
+						$found_localhost = true;
+						break;
+					}
+				}
+			}
+			if (! $found_localhost) {
+				$localhub = q("select hubloc.*, site.site_crypto, site.site_flags, site.site_version, site.site_project, site.site_dead from hubloc 
+					left join site on site_url = hubloc_url where hubloc_id_url = '%s' and hubloc_error = 0 and hubloc_deleted = 0",
+					dbesc(z_root() . '/channel/sys')
+				);
+				if ($localhub) {
+					$hubs = array_merge($hubs, $localhub);
+				}
+			}
+		}
  
-
-		if(! $r) {
+		if(! $hubs) {
 			logger('notifier: no hubs', LOGGER_NORMAL, LOG_NOTICE);
 			return;
 		}
 
-		$hubs = $r;
-
 		/**
 		 * Reduce the hubs to those that are unique. For zot hubs, we need to verify uniqueness by the sitekey, 
 		 * since it may have been a re-install which has not yet been detected and pruned.
@@ -556,9 +609,15 @@ class Notifier {
 		$keys    = []; // array of keys to check uniquness for zot hubs
 		$urls    = []; // array of urls to check uniqueness of hubs from other networks
 		$hub_env = []; // per-hub envelope so we don't broadcast the entire envelope to all
+		$dead    = []; // known dead hubs - report them as undeliverable
 
 		foreach($hubs as $hub) {
 
+			if (intval($hub['site_dead'])) {
+				$dead[] = $hub;
+				continue;
+			}
+
 			if($env_recips) {
 				foreach($env_recips as $er) {
 					if($hub['hubloc_hash'] === $er['hash']) {
@@ -630,7 +689,7 @@ class Notifier {
 			}
 
 			// singleton deliveries by definition 'not got zot'.
-            // Single deliveries are other federated networks (plugins) and we're essentially
+			// Single deliveries are other federated networks (plugins) and we're essentially
 			// delivering only to those that have this site url in their abook_instance
 			// and only from within a sync operation. This means if you post from a clone,
 			// and a connection is connected to one of your other clones; assuming that hub
@@ -725,7 +784,24 @@ class Notifier {
 						$packet = zot_build_packet($channel,'notify',$env, (($private) ? $hub['hubloc_sitekey'] : null), $hub['site_crypto'],$hash);
 
 					}
-				}	
+				}
+
+
+				// remove this after most hubs have updated to version 5.0
+				if(stripos($hub['site_project'], 'hubzilla') !== false && version_compare($hub['site_version'], '4.7.3', '<=')) {
+					if($encoded_item['type'] === 'mail') {
+						$encoded_item['from']['network'] = 'zot';
+						$encoded_item['from']['guid_sig'] = str_replace('sha256.', '', $encoded_item['from']['guid_sig']);
+					}
+					else {
+						$encoded_item['owner']['network'] = 'zot';
+						$encoded_item['owner']['guid_sig'] = str_replace('sha256.', '', $encoded_item['owner']['guid_sig']);
+						if(strpos($encoded_item['author']['url'], z_root()) === 0) {
+							$encoded_item['author']['network'] = 'zot';
+							$encoded_item['author']['guid_sig'] = str_replace('sha256.', '', $encoded_item['author']['guid_sig']);
+						}
+					}
+				}
 
 				queue_insert(
 					[
@@ -768,6 +844,24 @@ class Notifier {
 
 		logger('notifier: basic loop complete.', LOGGER_DEBUG);
 
+		if ($dead) {
+			foreach ($dead as $deceased) {
+				if (is_array($target_item) && (! $target_item['item_deleted']) && (! get_config('system','disable_dreport'))) {
+					q("insert into dreport ( dreport_mid, dreport_site, dreport_recip, dreport_name, dreport_result, dreport_time, dreport_xchan, dreport_queue ) 
+						values ( '%s', '%s','%s','%s','%s','%s','%s','%s' ) ",
+						dbesc($target_item['mid']),
+						dbesc($deceased['hubloc_host']),
+						dbesc($deceased['hubloc_host']),
+						dbesc($deceased['hubloc_host']),
+						dbesc('undeliverable/unresponsive site'),
+						dbesc(datetime_convert()),
+						dbesc($channel['channel_hash']),
+						dbesc(random_string(48))
+					);
+				}
+			}
+		}
+
 		call_hooks('notifier_end',$target_item);
 
 		logger('notifer: complete.');
diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php
index 2f06ec125..93a5412b0 100644
--- a/Zotlabs/Daemon/Onepoll.php
+++ b/Zotlabs/Daemon/Onepoll.php
@@ -2,6 +2,8 @@
 
 namespace Zotlabs\Daemon;
 
+use Zotlabs\Lib\Libzot;
+
 require_once('include/zot.php');
 require_once('include/socgraph.php');
 
@@ -76,7 +78,11 @@ class Onepoll {
 
 		// update permissions
 
-		$x = zot_refresh($contact,$importer);
+		if($contact['xchan_network'] === 'zot6')
+			$x = Libzot::refresh($contact,$importer);
+
+		if($contact['xchan_network'] === 'zot')
+			$x = zot_refresh($contact,$importer);
 
 		$responded = false;
 		$updated   = datetime_convert();