bring back externals (slightly different approach), rename zotfeed to outbox for consistency wit AP naming. warning: this commit requires update of addons!

6 files changed, 96 insertions, 225 deletions

diff --git a/Zotlabs/Daemon/Cron.php b/Zotlabs/Daemon/Cron.php
index c0a190c8e..6629491de 100644
--- a/Zotlabs/Daemon/Cron.php
+++ b/Zotlabs/Daemon/Cron.php
@@ -36,7 +36,6 @@ class Cron {
 		// run queue delivery process in the background
 
 		Master::Summon(array('Queue'));
-
 		Master::Summon(array('Poller'));
 
 		/**
@@ -206,10 +205,9 @@ class Cron {
 
 		// pull in some public posts
 
-/*		$disable_discover_tab = get_config('system', 'disable_discover_tab') || get_config('system', 'disable_discover_tab') === false;
+		$disable_discover_tab = get_config('system', 'disable_discover_tab') || get_config('system', 'disable_discover_tab') === false;
 		if (!$disable_discover_tab)
-			Master::Summon(array('Externals'));
-*/
+			Master::Summon(['Externals']);
 
 		$restart = false;
 
diff --git a/Zotlabs/Daemon/Externals.php b/Zotlabs/Daemon/Externals.php
index 064b3f71d..71589a4c6 100644
--- a/Zotlabs/Daemon/Externals.php
+++ b/Zotlabs/Daemon/Externals.php
@@ -3,6 +3,7 @@
 namespace Zotlabs\Daemon;
 
 use Zotlabs\Lib\Activity;
+use Zotlabs\Lib\Libzot;
 use Zotlabs\Lib\ActivityStreams;
 use Zotlabs\Lib\ASCollection;
 
@@ -31,25 +32,55 @@ class Externals {
 				$url = $arr['url'];
 			}
 			else {
+				$networks = ['zot6'];
+
+				if (plugin_is_installed('pubcrawl')) {
+					$networks[] = 'activitypub';
+				}
+
+				stringify_array_elms($networks);
+				$networks_str = implode(',', $networks);
+
 				$randfunc = db_getfunc('RAND');
 
 				// fixme this query does not deal with directory realms.
-
-				$r = q("select site_url, site_pull from site where site_url != '%s'
-						and site_flags != %d and site_type = %d
-						and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1",
+				//$r = q("select site_url, site_pull from site where site_url != '%s'
+						//and site_flags != %d and site_type = %d
+						//and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1",
+					//dbesc(z_root()),
+					//intval(DIRECTORY_MODE_STANDALONE),
+					//intval(SITE_TYPE_ZOT),
+					//dbesc('hubzilla%')
+				//);
+
+				$r = q("SELECT * FROM hubloc
+					LEFT JOIN abook ON abook_xchan = hubloc_hash
+					LEFT JOIN site ON site_url = hubloc_url WHERE
+					hubloc_network IN ( $networks_str ) AND
+					abook_xchan IS NULL AND
+					hubloc_url != '%s' AND
+					hubloc_updated > '%s' AND
+					hubloc_primary = 1 AND hubloc_deleted = 0 AND
+					site_dead = 0
+					ORDER BY $randfunc LIMIT 1",
 					dbesc(z_root()),
-					intval(DIRECTORY_MODE_STANDALONE),
-					intval(SITE_TYPE_ZOT),
-					dbesc('hubzilla%')
+					datetime_convert('UTC', 'UTC', 'now - 30 days')
 				);
-				if ($r)
-					$url = $r[0]['site_url'];
+
+				$contact = $r[0];
+
+				if ($contact) {
+					$url = $contact['hubloc_id_url'];
+				}
+			}
+
+			if (!$url) {
+				continue;
 			}
 
 			$blacklisted = false;
 
-			if (!check_siteallowed($url)) {
+			if (!check_siteallowed($contact['hubloc_url'])) {
 				logger('blacklisted site: ' . $url);
 				$blacklisted = true;
 			}
@@ -59,123 +90,65 @@ class Externals {
 			// make sure we can eventually break out if somebody blacklists all known sites
 
 			if ($blacklisted) {
-				if ($attempts > 20)
+				if ($attempts > 5)
 					break;
 				$attempts--;
 				continue;
 			}
 
-			if ($url) {
+			$cl = Activity::get_actor_collections($contact['hubloc_hash']);
+			if(empty($cl)) {
+				$cl = get_xconfig($contact['hubloc_hash'], 'activitypub', 'collections');
+			}
 
-				$max = intval(get_config('system', 'max_imported_posts', 30));
-				if (intval($max)) {
-					logger('externals: fetching outbox');
-
-					$feed_url = $url . '/zotfeed';
-					$obj      = new ASCollection($feed_url, $importer, 0, $max);
-					$messages = $obj->get();
-
-					if ($messages) {
-						foreach ($messages as $message) {
-							if (is_string($message)) {
-								$message = Activity::fetch($message, $importer);
-							}
-							$AS = new ActivityStreams($message);
-							if ($AS->is_valid() && is_array($AS->obj)) {
-								$item = Activity::decode_note($AS);
-								Activity::store($importer, $importer['xchan_hash'], $AS, $item, true);
-								$total++;
-							}
-						}
-					}
-					logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG);
+			if (is_array($cl) && array_key_exists('outbox', $cl)) {
+				$url = $cl['outbox'];
+			}
+			else {
+				$url = str_replace('/channel/', '/outbox/', $contact['hubloc_id_url']);
+				if ($url) {
+					$url .= '?top=1';
 				}
 			}
-		}
-		return;
-
-		/*		$total    = 0;
-				$attempts = 0;
-
-				logger('externals: startup', LOGGER_DEBUG);
-
-				// pull in some public posts
-
-				while ($total == 0 && $attempts < 3) {
-					$arr = ['url' => ''];
-					call_hooks('externals_url_select', $arr);
 
-					if ($arr['url']) {
-						$url = $arr['url'];
-					}
-					else {
-						$randfunc = db_getfunc('RAND');
-
-						// fixme this query does not deal with directory realms.
-
-						$r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d and site_dead = 0 order by $randfunc limit 1",
-							dbesc(z_root()),
-							intval(DIRECTORY_MODE_STANDALONE),
-							intval(SITE_TYPE_ZOT)
-						);
-						if ($r)
-							$url = $r[0]['site_url'];
-					}
-
-					$blacklisted = false;
+			if ($url) {
+				logger('fetching outbox: ' . $url);
 
-					if (!check_siteallowed($url)) {
-						logger('blacklisted site: ' . $url);
-						$blacklisted = true;
-					}
+				$obj      = new ASCollection($url, $importer, 0, 10);
+				$messages = $obj->get();
 
-					$attempts++;
+				if ($messages) {
+					foreach ($messages as $message) {
+						if (is_string($message)) {
+							$message = Activity::fetch($message, $importer);
+						}
 
-					// make sure we can eventually break out if somebody blacklists all known sites
+						if ($message['type'] !== 'Create') {
+							continue;
+						}
 
-					if ($blacklisted) {
-						if ($attempts > 20)
-							break;
-						$attempts--;
-						continue;
-					}
+						// make sure we only fetch top level items
+						if (isset($message['object']['inReplyTo'])) {
+							continue;
+						}
 
-					if ($url) {
-						if ($r[0]['site_pull'] > NULL_DATE)
-							$mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day'));
-						else {
-							$days = get_config('externals', 'since_days');
-							if ($days === false)
-								$days = 15;
-							$mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days'));
+						if ($contact['hubloc_network'] === 'zot6') {
+							Libzot::fetch_conversation($importer, $message['object']['id']);
+							$total++;
+							continue;
 						}
 
-						$feedurl = $url . '/zotfeed?f=&mindate=' . $mindate;
-
-						logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG);
-
-						$x = z_fetch_url($feedurl);
-						if (($x) && ($x['success'])) {
-
-							q("update site set site_pull = '%s' where site_url = '%s'",
-								dbesc(datetime_convert()),
-								dbesc($url)
-							);
-
-							$j = json_decode($x['body'], true);
-							if ($j['success'] && $j['messages']) {
-								$sys = get_sys_channel();
-								foreach ($j['messages'] as $message) {
-									// on these posts, clear any route info.
-									$message['route'] = '';
-									process_delivery(['hash' => 'undefined'], get_item_elements($message),
-										[['hash' => $sys['xchan_hash']]], false, true);
-									$total++;
-								}
-								logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG);
-							}
+						$AS = new ActivityStreams($message);
+						if ($AS->is_valid() && is_array($AS->obj)) {
+							$item = Activity::decode_note($AS);
+							Activity::store($importer, $contact['abook_xchan'], $AS, $item);
+							$total++;
 						}
 					}
-				}*/
+				}
+				logger('fetched messages count: ' . $total);
+			}
+		}
+		return;
 	}
 }
diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php
index d3294affb..e2a02ede4 100644
--- a/Zotlabs/Daemon/Onepoll.php
+++ b/Zotlabs/Daemon/Onepoll.php
@@ -135,7 +135,7 @@ class Onepoll {
 					$url = $cl['outbox'];
 				}
 				else {
-					$url = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']);
+					$url = str_replace('/poco/', '/outbox/', $contact['xchan_connurl']);
 				}
 
 				if ($url) {
diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php
index a0a9a7c7f..fe1e89851 100644
--- a/Zotlabs/Lib/Activity.php
+++ b/Zotlabs/Lib/Activity.php
@@ -2162,9 +2162,11 @@ class Activity {
 
 		if (array_key_exists('published', $act->data)) {
 			$s['created'] = datetime_convert('UTC', 'UTC', $act->data['published']);
+			$s['commented'] = $s['created'];
 		}
 		elseif (array_key_exists('published', $act->obj)) {
 			$s['created'] = datetime_convert('UTC', 'UTC', $act->obj['published']);
+			$s['commented'] = $s['created'];
 		}
 		if (array_key_exists('updated', $act->data)) {
 			$s['edited'] = datetime_convert('UTC', 'UTC', $act->data['updated']);
diff --git a/Zotlabs/Module/Zotfeed.php b/Zotlabs/Module/Zotfeed.php
index e47367036..0b4c3c007 100644
--- a/Zotlabs/Module/Zotfeed.php
+++ b/Zotlabs/Module/Zotfeed.php
@@ -1,124 +1,22 @@
 <?php
-
 namespace Zotlabs\Module;
 
-use App;
-use Zotlabs\Lib\Activity;
-use Zotlabs\Lib\ActivityStreams;
-use Zotlabs\Lib\Config;
-use Zotlabs\Lib\ThreadListener;
 use Zotlabs\Web\Controller;
-use Zotlabs\Web\HTTPSig;
 
 class Zotfeed extends Controller {
 
-	function init() {
-		if (ActivityStreams::is_as_request()) {
-
-			if (observer_prohibited(true)) {
-				killme();
-			}
-
-			$channel = channelx_by_nick(argv(1));
-			if (!$channel) {
-				killme();
-			}
-
-			if (intval($channel['channel_system'])) {
-				killme();
-			}
-
-			$sigdata = HTTPSig::verify(($_SERVER['REQUEST_METHOD'] === 'POST') ? file_get_contents('php://input') : EMPTY_STR);
-			if ($sigdata['portable_id'] && $sigdata['header_valid']) {
-				$portable_id = $sigdata['portable_id'];
-				if (!check_channelallowed($portable_id)) {
-					http_status_exit(403, 'Permission denied');
-				}
-				if (!check_siteallowed($sigdata['signer'])) {
-					http_status_exit(403, 'Permission denied');
-				}
-				observer_auth($portable_id);
-			}
-			elseif (Config::get('system', 'require_authenticated_fetch', false)) {
-				http_status_exit(403, 'Permission denied');
-			}
-
-			$observer_hash = get_observer_hash();
-
-			$params = [];
+	function post() {
 
-			$params['begin']     = ((x($_REQUEST, 'date_begin')) ? $_REQUEST['date_begin'] : NULL_DATE);
-			$params['end']       = ((x($_REQUEST, 'date_end')) ? $_REQUEST['date_end'] : '');
-			$params['type']      = 'json';
-			$params['pages']     = ((x($_REQUEST, 'pages')) ? intval($_REQUEST['pages']) : 0);
-			$params['top']       = ((x($_REQUEST, 'top')) ? intval($_REQUEST['top']) : 0);
-			$params['direction'] = ((x($_REQUEST, 'direction')) ? dbesc($_REQUEST['direction']) : 'desc'); // unimplemented
-			$params['cat']       = ((x($_REQUEST, 'cat')) ? escape_tags($_REQUEST['cat']) : '');
-			$params['compat']    = 1;
-
-			$total = items_fetch(
-				[
-					'total'      => true,
-					'wall'       => 1,
-					'datequery'  => $params['end'],
-					'datequery2' => $params['begin'],
-					'direction'  => dbesc($params['direction']),
-					'pages'      => $params['pages'],
-					'order'      => dbesc('post'),
-					'top'        => $params['top'],
-					'cat'        => $params['cat'],
-					'compat'     => $params['compat']
-				], $channel, $observer_hash, CLIENT_MODE_NORMAL, App::$module
-			);
-
-			if ($total) {
-				App::set_pager_total($total);
-				App::set_pager_itemspage(30);
-			}
+	}
 
-			if (App::$pager['unset'] && $total > 30) {
-				$ret = Activity::paged_collection_init($total, App::$query_string);
-			}
-			else {
+	function get() {
 
-				$items = items_fetch(
-					[
-						'wall'       => 1,
-						'datequery'  => $params['end'],
-						'datequery2' => $params['begin'],
-						'records'    => intval(App::$pager['itemspage']),
-						'start'      => intval(App::$pager['start']),
-						'direction'  => dbesc($params['direction']),
-						'pages'      => $params['pages'],
-						'order'      => dbesc('post'),
-						'top'        => $params['top'],
-						'cat'        => $params['cat'],
-						'compat'     => $params['compat']
-					], $channel, $observer_hash, CLIENT_MODE_NORMAL, App::$module
-				);
+		$outbox = new Outbox();
+		return $outbox->init();
 
-				if ($items && $observer_hash) {
+	}
 
-					// check to see if this observer is a connection. If not, register any items
-					// belonging to this channel for notification of deletion/expiration
+}
 
-					$x = q("select abook_id from abook where abook_channel = %d and abook_xchan = '%s'",
-						intval($channel['channel_id']),
-						dbesc($observer_hash)
-					);
-					if (!$x) {
-						foreach ($items as $item) {
-							if (strpos($item['mid'], z_root()) === 0) {
-								ThreadListener::store($item['mid'], $observer_hash);
-							}
-						}
-					}
-				}
 
-				$ret = Activity::encode_item_collection($items, App::$query_string, 'OrderedCollection', $total);
-			}
 
-			as_return_and_die($ret, $channel);
-		}
-	}
-}
diff --git a/boot.php b/boot.php
index 9fec3c482..cdba24f25 100644
--- a/boot.php
+++ b/boot.php
@@ -53,7 +53,7 @@ require_once('include/bbcode.php');
 require_once('include/items.php');
 
 define ( 'PLATFORM_NAME',           'hubzilla' );
-define ( 'STD_VERSION',             '6.3.3' );
+define ( 'STD_VERSION',             '6.3.4' );
 define ( 'ZOT_REVISION',            '6.0' );
 
 define ( 'DB_UPDATE_VERSION',       1248 );