diff options
Diffstat (limited to 'Zotlabs/Daemon/Externals.php')
| -rw-r--r-- | Zotlabs/Daemon/Externals.php | 193 |
1 files changed, 83 insertions, 110 deletions
diff --git a/Zotlabs/Daemon/Externals.php b/Zotlabs/Daemon/Externals.php index 064b3f71d..71589a4c6 100644 --- a/Zotlabs/Daemon/Externals.php +++ b/Zotlabs/Daemon/Externals.php @@ -3,6 +3,7 @@ namespace Zotlabs\Daemon; use Zotlabs\Lib\Activity; +use Zotlabs\Lib\Libzot; use Zotlabs\Lib\ActivityStreams; use Zotlabs\Lib\ASCollection; @@ -31,25 +32,55 @@ class Externals { $url = $arr['url']; } else { + $networks = ['zot6']; + + if (plugin_is_installed('pubcrawl')) { + $networks[] = 'activitypub'; + } + + stringify_array_elms($networks); + $networks_str = implode(',', $networks); + $randfunc = db_getfunc('RAND'); // fixme this query does not deal with directory realms. - - $r = q("select site_url, site_pull from site where site_url != '%s' - and site_flags != %d and site_type = %d - and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1", + //$r = q("select site_url, site_pull from site where site_url != '%s' + //and site_flags != %d and site_type = %d + //and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1", + //dbesc(z_root()), + //intval(DIRECTORY_MODE_STANDALONE), + //intval(SITE_TYPE_ZOT), + //dbesc('hubzilla%') + //); + + $r = q("SELECT * FROM hubloc + LEFT JOIN abook ON abook_xchan = hubloc_hash + LEFT JOIN site ON site_url = hubloc_url WHERE + hubloc_network IN ( $networks_str ) AND + abook_xchan IS NULL AND + hubloc_url != '%s' AND + hubloc_updated > '%s' AND + hubloc_primary = 1 AND hubloc_deleted = 0 AND + site_dead = 0 + ORDER BY $randfunc LIMIT 1", dbesc(z_root()), - intval(DIRECTORY_MODE_STANDALONE), - intval(SITE_TYPE_ZOT), - dbesc('hubzilla%') + datetime_convert('UTC', 'UTC', 'now - 30 days') ); - if ($r) - $url = $r[0]['site_url']; + + $contact = $r[0]; + + if ($contact) { + $url = $contact['hubloc_id_url']; + } + } + + if (!$url) { + continue; } $blacklisted = false; - if (!check_siteallowed($url)) { + if (!check_siteallowed($contact['hubloc_url'])) { logger('blacklisted site: ' . $url); $blacklisted = true; } @@ -59,123 +90,65 @@ class Externals { // make sure we can eventually break out if somebody blacklists all known sites if ($blacklisted) { - if ($attempts > 20) + if ($attempts > 5) break; $attempts--; continue; } - if ($url) { + $cl = Activity::get_actor_collections($contact['hubloc_hash']); + if(empty($cl)) { + $cl = get_xconfig($contact['hubloc_hash'], 'activitypub', 'collections'); + } - $max = intval(get_config('system', 'max_imported_posts', 30)); - if (intval($max)) { - logger('externals: fetching outbox'); - - $feed_url = $url . '/zotfeed'; - $obj = new ASCollection($feed_url, $importer, 0, $max); - $messages = $obj->get(); - - if ($messages) { - foreach ($messages as $message) { - if (is_string($message)) { - $message = Activity::fetch($message, $importer); - } - $AS = new ActivityStreams($message); - if ($AS->is_valid() && is_array($AS->obj)) { - $item = Activity::decode_note($AS); - Activity::store($importer, $importer['xchan_hash'], $AS, $item, true); - $total++; - } - } - } - logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG); + if (is_array($cl) && array_key_exists('outbox', $cl)) { + $url = $cl['outbox']; + } + else { + $url = str_replace('/channel/', '/outbox/', $contact['hubloc_id_url']); + if ($url) { + $url .= '?top=1'; } } - } - return; - - /* $total = 0; - $attempts = 0; - - logger('externals: startup', LOGGER_DEBUG); - - // pull in some public posts - - while ($total == 0 && $attempts < 3) { - $arr = ['url' => '']; - call_hooks('externals_url_select', $arr); - if ($arr['url']) { - $url = $arr['url']; - } - else { - $randfunc = db_getfunc('RAND'); - - // fixme this query does not deal with directory realms. - - $r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d and site_dead = 0 order by $randfunc limit 1", - dbesc(z_root()), - intval(DIRECTORY_MODE_STANDALONE), - intval(SITE_TYPE_ZOT) - ); - if ($r) - $url = $r[0]['site_url']; - } - - $blacklisted = false; + if ($url) { + logger('fetching outbox: ' . $url); - if (!check_siteallowed($url)) { - logger('blacklisted site: ' . $url); - $blacklisted = true; - } + $obj = new ASCollection($url, $importer, 0, 10); + $messages = $obj->get(); - $attempts++; + if ($messages) { + foreach ($messages as $message) { + if (is_string($message)) { + $message = Activity::fetch($message, $importer); + } - // make sure we can eventually break out if somebody blacklists all known sites + if ($message['type'] !== 'Create') { + continue; + } - if ($blacklisted) { - if ($attempts > 20) - break; - $attempts--; - continue; - } + // make sure we only fetch top level items + if (isset($message['object']['inReplyTo'])) { + continue; + } - if ($url) { - if ($r[0]['site_pull'] > NULL_DATE) - $mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day')); - else { - $days = get_config('externals', 'since_days'); - if ($days === false) - $days = 15; - $mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days')); + if ($contact['hubloc_network'] === 'zot6') { + Libzot::fetch_conversation($importer, $message['object']['id']); + $total++; + continue; } - $feedurl = $url . '/zotfeed?f=&mindate=' . $mindate; - - logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG); - - $x = z_fetch_url($feedurl); - if (($x) && ($x['success'])) { - - q("update site set site_pull = '%s' where site_url = '%s'", - dbesc(datetime_convert()), - dbesc($url) - ); - - $j = json_decode($x['body'], true); - if ($j['success'] && $j['messages']) { - $sys = get_sys_channel(); - foreach ($j['messages'] as $message) { - // on these posts, clear any route info. - $message['route'] = ''; - process_delivery(['hash' => 'undefined'], get_item_elements($message), - [['hash' => $sys['xchan_hash']]], false, true); - $total++; - } - logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG); - } + $AS = new ActivityStreams($message); + if ($AS->is_valid() && is_array($AS->obj)) { + $item = Activity::decode_note($AS); + Activity::store($importer, $contact['abook_xchan'], $AS, $item); + $total++; } } - }*/ + } + logger('fetched messages count: ' . $total); + } + } + return; } } |