What this checkin does is catch the case where a comment arrived and there's a missing top-level post to match it with. So we'll send a request back to the sender that you've never seen this thread and please send a fresh copy of the entire conversation to date. We could soon have posts in the matrix from different platforms from days gone by, which have been migrated into the modern world. We'll be polite and not deliver these to everybody. However, if someone comments on one of these antique threads we wouldn't be able to see it in our own matrix because we won't have a copy of the parent post. So this rectifies that situation. Be aware that item deletion may need to change to keep "hard deleted" items indefinitely so that they don't keep coming back. We'll have to null out the important data of the former item to accomplish the deletion aspect.

8 files changed, 180 insertions, 23 deletions

diff --git a/include/deliver.php b/include/deliver.php
index f4fae6061..8f6ba543d 100644
--- a/include/deliver.php
+++ b/include/deliver.php
@@ -38,15 +38,30 @@ function deliver_run($argv, $argc) {
 				continue;
 			}
 
-			if($r[0]['outq_posturl'] === z_root() . '/post') {
+			$notify = json_decode($r[0]['outq_notify'],true);
+
+			$sendtoweb = false;
+			if(array_key_exists('iv',$notify) && (! $r[0]['outq_msg']))
+				$sendtoweb = true;
+
+			if(($r[0]['outq_posturl'] === z_root() . '/post') && (! $sendtoweb)) {
 				logger('deliver: local delivery', LOGGER_DEBUG);
 				// local delivery
 				// we should probably batch these and save a few delivery processes
 				// If there is no outq_msg, this is a refresh_all message which does not require local handling
+				// also send 'request' packets to the webservice so it can decode the packet
 				if($r[0]['outq_msg']) {
-					$msg = array('body' => json_encode(array('pickup' => array(array('notify' => json_decode($r[0]['outq_notify'],true),'message' => json_decode($r[0]['outq_msg'],true))))));
-
-					zot_import($msg,z_root());
+					$m = json_decode($r[0]['outq_msg'],true);
+					if(array_key_exists('message_list',$m)) {
+						foreach($m['message_list'] as $mm) {
+							$msg = array('body' => json_encode(array('pickup' => array(array('notify' => $notify,'message' => $mm)))));
+							zot_import($msg,z_root());
+						}
+					}	
+					else {	
+						$msg = array('body' => json_encode(array('pickup' => array(array('notify' => $notify,'message' => $m)))));
+						zot_import($msg,z_root());
+					}
 					$r = q("delete from outq where outq_hash = '%s' limit 1",
 						dbesc($argv[$x])
 					);
diff --git a/include/items.php b/include/items.php
index 41c211953..d173e1a98 100755
--- a/include/items.php
+++ b/include/items.php
@@ -4249,14 +4249,24 @@ function fetch_post_tags($items,$link = false) {
 
 
 
-function zot_feed($uid,$observer_xchan,$mindate) {
+function zot_feed($uid,$observer_xchan,$arr) {
 
 
 	$result = array();
-	$mindate = datetime_convert('UTC','UTC',$mindate);
+	$mindate = null;
+	$message_id = null;
+
+	if(array_key_exists('mindate',$arr)) {
+		$mindate = datetime_convert('UTC','UTC',$arr['mindate']);
+	}
+
+	if(array_key_exists('message_id',$arr)) {
+		$message_id = $arr['message_id'];
+	}
+
+
 	if(! $mindate)
 		$mindate = NULL_DATE;
-
 	$mindate = dbesc($mindate);
 
 	logger('zot_feed: ' . $uid);
@@ -4278,6 +4288,11 @@ function zot_feed($uid,$observer_xchan,$mindate) {
 	else
 		$limit = " limit 0, 50 ";
 
+	if($message_id) {
+		$sql_extra .= " and mid = '" . dbesc($message_id) . "' ";
+		$limit = '';
+	}
+
 	$items = array();
 
 	if(is_sys_channel($uid)) {
diff --git a/include/notifier.php b/include/notifier.php
index 79a6886ad..57494885a 100644
--- a/include/notifier.php
+++ b/include/notifier.php
@@ -58,6 +58,7 @@ require_once('include/html2plain.php');
  *       expire                 channel_id
  *       relay					item_id (item was relayed to owner, we will deliver it as owner)
  *       location               channel_id
+ *       request                channel_id            xchan_hash             message_id
  *
  */
 
@@ -142,6 +143,7 @@ function notifier_run($argv, $argc){
 
 
 	$expire = false;
+	$request = false;
 	$mail = false;
 	$fsuggest = false;
 	$top_level = false;
@@ -175,6 +177,23 @@ function notifier_run($argv, $argc){
 			$channel = $s[0];
 
 	}
+	elseif($cmd === 'request') {
+		$channel_id = $item_id;
+		$xchan = $argv[3];
+		$request_message_id = $argv[4];
+
+		$s = q("select * from channel where channel_id = %d limit 1",
+			intval($channel_id)
+		);
+		if($s)
+			$channel = $s[0];
+
+		$private = true;
+		$recipients[] = $xchan;
+		$packet_type = 'request';
+		$normal_mode = false;
+
+	}
 	elseif($cmd === 'expire') {
 
 		// FIXME
@@ -550,6 +569,7 @@ function notifier_run($argv, $argc){
 				'mail' => $mail,
 				'location' => $location,
 				'fsuggest' => $fsuggest,
+				'request' => $request,
 				'normal_mode' => $normal_mode,
 				'packet_type' => $packet_type,
 				'walltowall' => $walltowall
@@ -579,6 +599,21 @@ function notifier_run($argv, $argc){
 				dbesc('')
 			);
 		}
+		elseif($packet_type === 'request') {
+			$n = zot_build_packet($channel,'request',$env_recips,$hub['hubloc_sitekey'],$hash,array('message_id' => $request_message_id));
+			q("insert into outq ( outq_hash, outq_account, outq_channel, outq_driver, outq_posturl, outq_async, outq_created, outq_updated, outq_notify, outq_msg ) values ( '%s', %d, %d, '%s', '%s', %d, '%s', '%s', '%s', '%s' )",
+				dbesc($hash),
+				intval($channel['channel_account_id']),
+				intval($channel['channel_id']),
+				dbesc('zot'),
+				dbesc($hub['hubloc_callback']),
+				intval(1),
+				dbesc(datetime_convert()),
+				dbesc(datetime_convert()),
+				dbesc($n),
+				dbesc('')
+			);
+		}
 		else {
 			$n = zot_build_packet($channel,'notify',$env_recips,(($private) ? $hub['hubloc_sitekey'] : null),$hash);
 			q("insert into outq ( outq_hash, outq_account, outq_channel, outq_driver, outq_posturl, outq_async, outq_created, outq_updated, outq_notify, outq_msg ) values ( '%s', %d, %d, '%s', '%s', %d, '%s', '%s', '%s', '%s' )",
diff --git a/include/onepoll.php b/include/onepoll.php
index 98d52db93..7a81282b3 100644
--- a/include/onepoll.php
+++ b/include/onepoll.php
@@ -15,12 +15,6 @@ function onepoll_run($argv, $argc){
 
 	logger('onepoll: start');
 	
-	$manual_id  = 0;
-	$generation = 0;
-
-	$force      = false;
-	$restart    = false;
-
 	if(($argc > 1) && (intval($argv[1])))
 		$contact_id = intval($argv[1]);
 
@@ -28,7 +22,7 @@ function onepoll_run($argv, $argc){
 		logger('onepoll: no contact');
 		return;
 	}
-	
+
 	$d = datetime_convert();
 
 	$contacts = q("SELECT abook.*, xchan.*, account.*
@@ -90,7 +84,8 @@ function onepoll_run($argv, $argc){
 	$x = zot_refresh($contact,$importer);
 
 	$responded = false;
-	$updated = datetime_convert();
+	$updated   = datetime_convert();
+	$connected = datetime_convert();
 	if(! $x) {
 		// mark for death by not updating abook_connected, this is caught in include/poller.php
 		q("update abook set abook_updated = '%s' where abook_id = %d limit 1",
@@ -101,7 +96,7 @@ function onepoll_run($argv, $argc){
 	else {
 		q("update abook set abook_updated = '%s', abook_connected = '%s' where abook_id = %d limit 1",
 			dbesc($updated),
-			dbesc($updated),
+			dbesc($connected),
 			intval($contact['abook_id'])
 		);
 		$responded = true;
@@ -120,7 +115,9 @@ function onepoll_run($argv, $argc){
 		if($fetch_feed) {
 
 			$feedurl = str_replace('/poco/','/zotfeed/',$contact['xchan_connurl']);		
-			$x = z_fetch_url($feedurl . '?f=&mindate=' . urlencode($last_update));
+			$feedurl .= '?f=&mindate=' . urlencode($last_update);
+
+			$x = z_fetch_url($feedurl);
 
 			logger('feed_update: ' . print_r($x,true), LOGGER_DATA);
 
@@ -143,8 +140,8 @@ function onepoll_run($argv, $argc){
 		}
 	}
 			
-	// fetch some items
-	// set last updated timestamp
+
+	// update the poco details for this connection
 
 	if($contact['xchan_connurl']) {	
 		$r = q("SELECT xlink_id from xlink 
diff --git a/include/zot.php b/include/zot.php
index e96eae10d..3a5139755 100644
--- a/include/zot.php
+++ b/include/zot.php
@@ -104,7 +104,7 @@ function zot_get_hublocs($hash) {
  * @returns string json encoded zot packet
  */
 
-function zot_build_packet($channel,$type = 'notify',$recipients = null, $remote_key = null, $secret = null) {
+function zot_build_packet($channel,$type = 'notify',$recipients = null, $remote_key = null, $secret = null, $extra = null) {
 
 	$data = array(
 		'type' => $type,
@@ -129,6 +129,12 @@ function zot_build_packet($channel,$type = 'notify',$recipients = null, $remote_
 		$data['secret_sig'] = base64url_encode(rsa_sign($secret,$channel['channel_prvkey']));
 	}
 
+	if($extra) {
+		foreach($extra as $k => $v)
+			$data[$k] = $v;
+	}
+
+
 	logger('zot_build_packet: ' . print_r($data,true), LOGGER_DATA);
 
 	// Hush-hush ultra top-secret mode
@@ -1436,6 +1442,13 @@ function process_delivery($sender,$arr,$deliveries,$relay,$public = false) {
 			);
 			if(! $r) {
 				$result[] = array($d['hash'],'comment parent not found',$channel['channel_name'] . ' <' . $channel['channel_address'] . '@' . get_app()->get_hostname() . '>',$arr['mid']);
+
+				// we don't seem to have a copy of this conversation or at least the parent - request a copy of the entire conversation to date.
+				// Don't do this if it's a relay post as we're the ones who are supposed to have the copy and we don't want the request to loop.
+
+				if(! $relay)
+					proc_run('php', 'include/notifier.php', 'request', $channel['channel_id'], $sender['hash'], $arr['parent_mid']);
+
 				continue;
 			}
 			if($relay) {
@@ -2793,3 +2806,71 @@ function import_author_zot($x) {
 	return false;
 }
 
+function zot_process_message_request($data) {
+	$ret = array('success' => false);
+	if(! $data['message_id']) {
+		$ret['message'] = 'no message_id';
+		logger('no message_id');
+		return $ret;
+	}
+
+	$sender = $data['sender'];
+	$sender_hash = make_xchan_hash($sender['guid'],$sender['guid_sig']);
+
+	$arr = $data['recipients'][0];
+	$recip_hash = make_xchan_hash($arr['guid'],$arr['guid_sig']);
+	$c = q("select * from channel left join xchan on channel_hash = xchan_hash where channel_hash = '%s' limit 1",
+		dbesc($recip_hash)
+	);
+	if(! $c) {
+		logger('recipient channel not found.');
+		$ret['message'] .= 'recipient not found.' . EOL;
+		return $ret;
+	}
+	$messages = zot_feed($c[0]['channel_id'],$sender_hash,array('message_id' => $data['message_id']));
+	if($messages) {
+		$env_recips = null;
+
+		$r = q("select hubloc_guid, hubloc_url, hubloc_sitekey, hubloc_network, hubloc_flags, hubloc_callback, hubloc_host 
+			from hubloc where hubloc_hash = '" . dbesc($sender_hash) . "' and not (hubloc_flags & %d)
+			and not (hubloc_status & %d) group by hubloc_sitekey",
+            intval(HUBLOC_FLAGS_DELETED),
+            intval(HUBLOC_OFFLINE)
+        );
+	    if(! $r) {
+	        logger('no hubs');
+    	    return $ret;
+    	}
+		$hubs = $r;
+		$hublist = array();
+		$keys = array();
+
+		$private = ((array_key_exists('flags',$messages[0]) && in_array('private',$messages[0]['flags'])) ? true : false);
+		if($private)
+			$env_recips = array('guid' => $sender['guid'],'guid_sig' => $sender['guid_sig'],'hash' => $sender_hash);
+		$data_packet = json_encode(array('message_list' => $messages));
+		
+		foreach($hubs as $hub) {
+			$hash = random_string();
+			$n = zot_build_packet($c[0],'notify',$env_recips,(($private) ? $hub['hubloc_sitekey'] : null),$hash);
+			q("insert into outq ( outq_hash, outq_account, outq_channel, outq_driver, outq_posturl, outq_async, 
+				outq_created, outq_updated, outq_notify, outq_msg ) 
+				values ( '%s', %d, %d, '%s', '%s', %d, '%s', '%s', '%s', '%s' )",
+				dbesc($hash),
+				intval($c[0]['channel_account_id']),
+				intval($c[0]['channel_id']),
+				dbesc('zot'),
+				dbesc($hub['hubloc_callback']),
+				intval(1),
+				dbesc(datetime_convert()),
+				dbesc(datetime_convert()),
+				dbesc($n),
+				dbesc($data_packet)
+			);
+			proc_run('php','include/deliver.php',$hash);
+		}
+
+	}
+	$ret['success'] = true;
+	return $ret;
+}
diff --git a/mod/post.php b/mod/post.php
index d62233ca1..8ffd3b5ad 100644
--- a/mod/post.php
+++ b/mod/post.php
@@ -507,6 +507,7 @@ function post_post(&$a) {
 		json_return_and_die($ret);
 	}
 
+
 	if($msgtype === 'pickup') {
 
 		/**
@@ -597,7 +598,14 @@ function post_post(&$a) {
 			$ret['success'] = true;
 			$ret['pickup'] = array();
 			foreach($r as $rr) {
-				$ret['pickup'][] = array('notify' => json_decode($rr['outq_notify'],true),'message' => json_decode($rr['outq_msg'],true));
+				$x = json_decode($rr['outq_msg'],true);
+
+				if(array_key_exists('message_list',$x)) {
+					foreach($x['message_list'] as $xx)
+						$ret['pickup'][] = array('notify' => json_decode($rr['outq_notify'],true),'message' => $xx);
+				}
+				else
+					$ret['pickup'][] = array('notify' => json_decode($rr['outq_notify'],true),'message' => $x);
 
 				$x = q("delete from outq where outq_hash = '%s' limit 1",
 					dbesc($rr['outq_hash'])
@@ -796,6 +804,12 @@ function post_post(&$a) {
 		json_return_and_die($ret);
 	}
 
+	if($msgtype === 'request') {
+		// request a particular post/conversation by message_id
+		$x = zot_process_message_request($data);
+		json_return_and_die($x);		
+	}
+
 
 	if($msgtype === 'purge') {
 		if($recipients) {
diff --git a/mod/zotfeed.php b/mod/zotfeed.php
index 480e886cd..c730e4162 100644
--- a/mod/zotfeed.php
+++ b/mod/zotfeed.php
@@ -38,7 +38,7 @@ function zotfeed_init(&$a) {
 
 	logger('zotfeed request: ' . $r[0]['channel_name'], LOGGER_DEBUG);
 
-	$result['messages'] = zot_feed($r[0]['channel_id'],$observer['xchan_hash'],$mindate);
+	$result['messages'] = zot_feed($r[0]['channel_id'],$observer['xchan_hash'],array('mindate' => $mindate));
 	$result['success'] = true;
 	json_return_and_die($result);
 
diff --git a/version.inc b/version.inc
index 5056907a7..e5dbf4259 100644
--- a/version.inc
+++ b/version.inc
@@ -1 +1 @@
-2014-10-29.843
+2014-10-30.844