diff options
| -rw-r--r-- | Zotlabs/Daemon/Cron.php | 6 | ||||
| -rw-r--r-- | Zotlabs/Daemon/Externals.php | 193 | ||||
| -rw-r--r-- | Zotlabs/Daemon/Onepoll.php | 2 | ||||
| -rw-r--r-- | Zotlabs/Lib/Activity.php | 2 | ||||
| -rw-r--r-- | Zotlabs/Module/Zotfeed.php | 116 | ||||
| -rw-r--r-- | boot.php | 2 |
6 files changed, 96 insertions, 225 deletions
diff --git a/Zotlabs/Daemon/Cron.php b/Zotlabs/Daemon/Cron.php index c0a190c8e..6629491de 100644 --- a/Zotlabs/Daemon/Cron.php +++ b/Zotlabs/Daemon/Cron.php @@ -36,7 +36,6 @@ class Cron { // run queue delivery process in the background Master::Summon(array('Queue')); - Master::Summon(array('Poller')); /** @@ -206,10 +205,9 @@ class Cron { // pull in some public posts -/* $disable_discover_tab = get_config('system', 'disable_discover_tab') || get_config('system', 'disable_discover_tab') === false; + $disable_discover_tab = get_config('system', 'disable_discover_tab') || get_config('system', 'disable_discover_tab') === false; if (!$disable_discover_tab) - Master::Summon(array('Externals')); -*/ + Master::Summon(['Externals']); $restart = false; diff --git a/Zotlabs/Daemon/Externals.php b/Zotlabs/Daemon/Externals.php index 064b3f71d..71589a4c6 100644 --- a/Zotlabs/Daemon/Externals.php +++ b/Zotlabs/Daemon/Externals.php @@ -3,6 +3,7 @@ namespace Zotlabs\Daemon; use Zotlabs\Lib\Activity; +use Zotlabs\Lib\Libzot; use Zotlabs\Lib\ActivityStreams; use Zotlabs\Lib\ASCollection; @@ -31,25 +32,55 @@ class Externals { $url = $arr['url']; } else { + $networks = ['zot6']; + + if (plugin_is_installed('pubcrawl')) { + $networks[] = 'activitypub'; + } + + stringify_array_elms($networks); + $networks_str = implode(',', $networks); + $randfunc = db_getfunc('RAND'); // fixme this query does not deal with directory realms. - - $r = q("select site_url, site_pull from site where site_url != '%s' - and site_flags != %d and site_type = %d - and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1", + //$r = q("select site_url, site_pull from site where site_url != '%s' + //and site_flags != %d and site_type = %d + //and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1", + //dbesc(z_root()), + //intval(DIRECTORY_MODE_STANDALONE), + //intval(SITE_TYPE_ZOT), + //dbesc('hubzilla%') + //); + + $r = q("SELECT * FROM hubloc + LEFT JOIN abook ON abook_xchan = hubloc_hash + LEFT JOIN site ON site_url = hubloc_url WHERE + hubloc_network IN ( $networks_str ) AND + abook_xchan IS NULL AND + hubloc_url != '%s' AND + hubloc_updated > '%s' AND + hubloc_primary = 1 AND hubloc_deleted = 0 AND + site_dead = 0 + ORDER BY $randfunc LIMIT 1", dbesc(z_root()), - intval(DIRECTORY_MODE_STANDALONE), - intval(SITE_TYPE_ZOT), - dbesc('hubzilla%') + datetime_convert('UTC', 'UTC', 'now - 30 days') ); - if ($r) - $url = $r[0]['site_url']; + + $contact = $r[0]; + + if ($contact) { + $url = $contact['hubloc_id_url']; + } + } + + if (!$url) { + continue; } $blacklisted = false; - if (!check_siteallowed($url)) { + if (!check_siteallowed($contact['hubloc_url'])) { logger('blacklisted site: ' . $url); $blacklisted = true; } @@ -59,123 +90,65 @@ class Externals { // make sure we can eventually break out if somebody blacklists all known sites if ($blacklisted) { - if ($attempts > 20) + if ($attempts > 5) break; $attempts--; continue; } - if ($url) { + $cl = Activity::get_actor_collections($contact['hubloc_hash']); + if(empty($cl)) { + $cl = get_xconfig($contact['hubloc_hash'], 'activitypub', 'collections'); + } - $max = intval(get_config('system', 'max_imported_posts', 30)); - if (intval($max)) { - logger('externals: fetching outbox'); - - $feed_url = $url . '/zotfeed'; - $obj = new ASCollection($feed_url, $importer, 0, $max); - $messages = $obj->get(); - - if ($messages) { - foreach ($messages as $message) { - if (is_string($message)) { - $message = Activity::fetch($message, $importer); - } - $AS = new ActivityStreams($message); - if ($AS->is_valid() && is_array($AS->obj)) { - $item = Activity::decode_note($AS); - Activity::store($importer, $importer['xchan_hash'], $AS, $item, true); - $total++; - } - } - } - logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG); + if (is_array($cl) && array_key_exists('outbox', $cl)) { + $url = $cl['outbox']; + } + else { + $url = str_replace('/channel/', '/outbox/', $contact['hubloc_id_url']); + if ($url) { + $url .= '?top=1'; } } - } - return; - - /* $total = 0; - $attempts = 0; - - logger('externals: startup', LOGGER_DEBUG); - - // pull in some public posts - - while ($total == 0 && $attempts < 3) { - $arr = ['url' => '']; - call_hooks('externals_url_select', $arr); - if ($arr['url']) { - $url = $arr['url']; - } - else { - $randfunc = db_getfunc('RAND'); - - // fixme this query does not deal with directory realms. - - $r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d and site_dead = 0 order by $randfunc limit 1", - dbesc(z_root()), - intval(DIRECTORY_MODE_STANDALONE), - intval(SITE_TYPE_ZOT) - ); - if ($r) - $url = $r[0]['site_url']; - } - - $blacklisted = false; + if ($url) { + logger('fetching outbox: ' . $url); - if (!check_siteallowed($url)) { - logger('blacklisted site: ' . $url); - $blacklisted = true; - } + $obj = new ASCollection($url, $importer, 0, 10); + $messages = $obj->get(); - $attempts++; + if ($messages) { + foreach ($messages as $message) { + if (is_string($message)) { + $message = Activity::fetch($message, $importer); + } - // make sure we can eventually break out if somebody blacklists all known sites + if ($message['type'] !== 'Create') { + continue; + } - if ($blacklisted) { - if ($attempts > 20) - break; - $attempts--; - continue; - } + // make sure we only fetch top level items + if (isset($message['object']['inReplyTo'])) { + continue; + } - if ($url) { - if ($r[0]['site_pull'] > NULL_DATE) - $mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day')); - else { - $days = get_config('externals', 'since_days'); - if ($days === false) - $days = 15; - $mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days')); + if ($contact['hubloc_network'] === 'zot6') { + Libzot::fetch_conversation($importer, $message['object']['id']); + $total++; + continue; } - $feedurl = $url . '/zotfeed?f=&mindate=' . $mindate; - - logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG); - - $x = z_fetch_url($feedurl); - if (($x) && ($x['success'])) { - - q("update site set site_pull = '%s' where site_url = '%s'", - dbesc(datetime_convert()), - dbesc($url) - ); - - $j = json_decode($x['body'], true); - if ($j['success'] && $j['messages']) { - $sys = get_sys_channel(); - foreach ($j['messages'] as $message) { - // on these posts, clear any route info. - $message['route'] = ''; - process_delivery(['hash' => 'undefined'], get_item_elements($message), - [['hash' => $sys['xchan_hash']]], false, true); - $total++; - } - logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG); - } + $AS = new ActivityStreams($message); + if ($AS->is_valid() && is_array($AS->obj)) { + $item = Activity::decode_note($AS); + Activity::store($importer, $contact['abook_xchan'], $AS, $item); + $total++; } } - }*/ + } + logger('fetched messages count: ' . $total); + } + } + return; } } diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index d3294affb..e2a02ede4 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -135,7 +135,7 @@ class Onepoll { $url = $cl['outbox']; } else { - $url = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']); + $url = str_replace('/poco/', '/outbox/', $contact['xchan_connurl']); } if ($url) { diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php index a0a9a7c7f..fe1e89851 100644 --- a/Zotlabs/Lib/Activity.php +++ b/Zotlabs/Lib/Activity.php @@ -2162,9 +2162,11 @@ class Activity { if (array_key_exists('published', $act->data)) { $s['created'] = datetime_convert('UTC', 'UTC', $act->data['published']); + $s['commented'] = $s['created']; } elseif (array_key_exists('published', $act->obj)) { $s['created'] = datetime_convert('UTC', 'UTC', $act->obj['published']); + $s['commented'] = $s['created']; } if (array_key_exists('updated', $act->data)) { $s['edited'] = datetime_convert('UTC', 'UTC', $act->data['updated']); diff --git a/Zotlabs/Module/Zotfeed.php b/Zotlabs/Module/Zotfeed.php index e47367036..0b4c3c007 100644 --- a/Zotlabs/Module/Zotfeed.php +++ b/Zotlabs/Module/Zotfeed.php @@ -1,124 +1,22 @@ <?php - namespace Zotlabs\Module; -use App; -use Zotlabs\Lib\Activity; -use Zotlabs\Lib\ActivityStreams; -use Zotlabs\Lib\Config; -use Zotlabs\Lib\ThreadListener; use Zotlabs\Web\Controller; -use Zotlabs\Web\HTTPSig; class Zotfeed extends Controller { - function init() { - if (ActivityStreams::is_as_request()) { - - if (observer_prohibited(true)) { - killme(); - } - - $channel = channelx_by_nick(argv(1)); - if (!$channel) { - killme(); - } - - if (intval($channel['channel_system'])) { - killme(); - } - - $sigdata = HTTPSig::verify(($_SERVER['REQUEST_METHOD'] === 'POST') ? file_get_contents('php://input') : EMPTY_STR); - if ($sigdata['portable_id'] && $sigdata['header_valid']) { - $portable_id = $sigdata['portable_id']; - if (!check_channelallowed($portable_id)) { - http_status_exit(403, 'Permission denied'); - } - if (!check_siteallowed($sigdata['signer'])) { - http_status_exit(403, 'Permission denied'); - } - observer_auth($portable_id); - } - elseif (Config::get('system', 'require_authenticated_fetch', false)) { - http_status_exit(403, 'Permission denied'); - } - - $observer_hash = get_observer_hash(); - - $params = []; + function post() { - $params['begin'] = ((x($_REQUEST, 'date_begin')) ? $_REQUEST['date_begin'] : NULL_DATE); - $params['end'] = ((x($_REQUEST, 'date_end')) ? $_REQUEST['date_end'] : ''); - $params['type'] = 'json'; - $params['pages'] = ((x($_REQUEST, 'pages')) ? intval($_REQUEST['pages']) : 0); - $params['top'] = ((x($_REQUEST, 'top')) ? intval($_REQUEST['top']) : 0); - $params['direction'] = ((x($_REQUEST, 'direction')) ? dbesc($_REQUEST['direction']) : 'desc'); // unimplemented - $params['cat'] = ((x($_REQUEST, 'cat')) ? escape_tags($_REQUEST['cat']) : ''); - $params['compat'] = 1; - - $total = items_fetch( - [ - 'total' => true, - 'wall' => 1, - 'datequery' => $params['end'], - 'datequery2' => $params['begin'], - 'direction' => dbesc($params['direction']), - 'pages' => $params['pages'], - 'order' => dbesc('post'), - 'top' => $params['top'], - 'cat' => $params['cat'], - 'compat' => $params['compat'] - ], $channel, $observer_hash, CLIENT_MODE_NORMAL, App::$module - ); - - if ($total) { - App::set_pager_total($total); - App::set_pager_itemspage(30); - } + } - if (App::$pager['unset'] && $total > 30) { - $ret = Activity::paged_collection_init($total, App::$query_string); - } - else { + function get() { - $items = items_fetch( - [ - 'wall' => 1, - 'datequery' => $params['end'], - 'datequery2' => $params['begin'], - 'records' => intval(App::$pager['itemspage']), - 'start' => intval(App::$pager['start']), - 'direction' => dbesc($params['direction']), - 'pages' => $params['pages'], - 'order' => dbesc('post'), - 'top' => $params['top'], - 'cat' => $params['cat'], - 'compat' => $params['compat'] - ], $channel, $observer_hash, CLIENT_MODE_NORMAL, App::$module - ); + $outbox = new Outbox(); + return $outbox->init(); - if ($items && $observer_hash) { + } - // check to see if this observer is a connection. If not, register any items - // belonging to this channel for notification of deletion/expiration +} - $x = q("select abook_id from abook where abook_channel = %d and abook_xchan = '%s'", - intval($channel['channel_id']), - dbesc($observer_hash) - ); - if (!$x) { - foreach ($items as $item) { - if (strpos($item['mid'], z_root()) === 0) { - ThreadListener::store($item['mid'], $observer_hash); - } - } - } - } - $ret = Activity::encode_item_collection($items, App::$query_string, 'OrderedCollection', $total); - } - as_return_and_die($ret, $channel); - } - } -} @@ -53,7 +53,7 @@ require_once('include/bbcode.php'); require_once('include/items.php'); define ( 'PLATFORM_NAME', 'hubzilla' ); -define ( 'STD_VERSION', '6.3.3' ); +define ( 'STD_VERSION', '6.3.4' ); define ( 'ZOT_REVISION', '6.0' ); define ( 'DB_UPDATE_VERSION', 1248 ); |