From 3836e75c89573baca6b029a5fd36401696b86e8d Mon Sep 17 00:00:00 2001 From: Max Kostikov Date: Thu, 21 Jan 2021 09:08:53 +0100 Subject: Implement SQL query background caching --- Zotlabs/Daemon/Cache_query.php | 34 ++++++++++++++++++++++++++++++++++ Zotlabs/Daemon/Expire.php | 8 ++++---- 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 Zotlabs/Daemon/Cache_query.php (limited to 'Zotlabs/Daemon') diff --git a/Zotlabs/Daemon/Cache_query.php b/Zotlabs/Daemon/Cache_query.php new file mode 100644 index 000000000..18d19cdf2 --- /dev/null +++ b/Zotlabs/Daemon/Cache_query.php @@ -0,0 +1,34 @@ + Date: Fri, 22 Jan 2021 10:06:50 +0000 Subject: implement externals via zot6 and zotfeed - part 1 --- Zotlabs/Daemon/Convo.php | 58 +++++++++++++++++ Zotlabs/Daemon/Cron.php | 6 +- Zotlabs/Daemon/Externals.php | 148 +++++++++++++++++++++++++++++++++---------- 3 files changed, 175 insertions(+), 37 deletions(-) create mode 100644 Zotlabs/Daemon/Convo.php (limited to 'Zotlabs/Daemon') diff --git a/Zotlabs/Daemon/Convo.php b/Zotlabs/Daemon/Convo.php new file mode 100644 index 000000000..f7478d778 --- /dev/null +++ b/Zotlabs/Daemon/Convo.php @@ -0,0 +1,58 @@ +get(); + + if ($messages) { + foreach ($messages as $message) { + if (is_string($message)) { + $message = Activity::fetch($message, $channel); + } + // set client flag because comments will probably just be objects and not full blown activities + // and that lets us use implied_create + $AS = new ActivityStreams($message); + if ($AS->is_valid() && is_array($AS->obj)) { + $item = Activity::decode_note($AS, true); + Activity::store($channel, $contact['abook_xchan'], $AS, $item); + } + } + } + } +} diff --git a/Zotlabs/Daemon/Cron.php b/Zotlabs/Daemon/Cron.php index cd7a710d3..2c0422ddd 100644 --- a/Zotlabs/Daemon/Cron.php +++ b/Zotlabs/Daemon/Cron.php @@ -201,7 +201,7 @@ class Cron { require_once('include/photo/photo_driver.php'); foreach ($r as $rr) { $photos = import_xchan_photo($rr['xchan_photo_l'], $rr['xchan_hash'], false, true); - $x = q("update xchan set xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s' + q("update xchan set xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s' where xchan_hash = '%s'", dbesc($photos[0]), dbesc($photos[1]), @@ -219,8 +219,6 @@ class Cron { if (!$disable_discover_tab) Master::Summon(array('Externals')); - $generation = 0; - $restart = false; if (($argc > 1) && ($argv[1] == 'restart')) { @@ -232,8 +230,6 @@ class Cron { reload_plugins(); - $d = datetime_convert(); - // TODO check to see if there are any cronhooks before wasting a process if (!$restart) diff --git a/Zotlabs/Daemon/Externals.php b/Zotlabs/Daemon/Externals.php index 3d6b0bd7b..da05cad18 100644 --- a/Zotlabs/Daemon/Externals.php +++ b/Zotlabs/Daemon/Externals.php @@ -2,7 +2,10 @@ namespace Zotlabs\Daemon; -require_once('include/zot.php'); +use Zotlabs\Lib\Activity; +use Zotlabs\Lib\ActivityStreams; +use Zotlabs\Lib\ASCollection; + require_once('include/channel.php'); @@ -10,6 +13,9 @@ class Externals { static public function run($argc, $argv) { + logger('externals: start'); + + $importer = get_sys_channel(); $total = 0; $attempts = 0; @@ -27,12 +33,15 @@ class Externals { else { $randfunc = db_getfunc('RAND'); - // fixme this query does not deal with directory realms. + // fixme this query does not deal with directory realms. - $r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d and site_dead = 0 order by $randfunc limit 1", + $r = q("select site_url, site_pull from site where site_url != '%s' + and site_flags != %d and site_type = %d + and site_dead = 0 and site_project like '%s' and site_version > '5.3.1' order by $randfunc limit 1", dbesc(z_root()), intval(DIRECTORY_MODE_STANDALONE), - intval(SITE_TYPE_ZOT) + intval(SITE_TYPE_ZOT), + dbesc('hubzilla%') ); if ($r) $url = $r[0]['site_url']; @@ -57,41 +66,116 @@ class Externals { } if ($url) { - if ($r[0]['site_pull'] > NULL_DATE) - $mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day')); - else { - $days = get_config('externals', 'since_days'); - if ($days === false) - $days = 15; - $mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days')); + + $max = intval(get_config('system', 'max_imported_posts', 30)); + if (intval($max)) { + logger('externals: fetching outbox'); + + $feed_url = $url . '/zotfeed'; + $obj = new ASCollection($feed_url, $importer, 0, $max); + $messages = $obj->get(); + + if ($messages) { + foreach ($messages as $message) { + if (is_string($message)) { + $message = Activity::fetch($message, $importer); + } + $AS = new ActivityStreams($message); + if ($AS->is_valid() && is_array($AS->obj)) { + $item = Activity::decode_note($AS); + Activity::store($importer, $importer['xchan_hash'], $AS, $item, true); + $total++; + } + } + } + logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG); } + } + } + return; + + /* $total = 0; + $attempts = 0; + + logger('externals: startup', LOGGER_DEBUG); + + // pull in some public posts + + while ($total == 0 && $attempts < 3) { + $arr = ['url' => '']; + call_hooks('externals_url_select', $arr); - $feedurl = $url . '/zotfeed?f=&mindate=' . $mindate; + if ($arr['url']) { + $url = $arr['url']; + } + else { + $randfunc = db_getfunc('RAND'); + + // fixme this query does not deal with directory realms. + + $r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d and site_dead = 0 order by $randfunc limit 1", + dbesc(z_root()), + intval(DIRECTORY_MODE_STANDALONE), + intval(SITE_TYPE_ZOT) + ); + if ($r) + $url = $r[0]['site_url']; + } + + $blacklisted = false; - logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG); + if (!check_siteallowed($url)) { + logger('blacklisted site: ' . $url); + $blacklisted = true; + } + + $attempts++; - $x = z_fetch_url($feedurl); - if (($x) && ($x['success'])) { + // make sure we can eventually break out if somebody blacklists all known sites - q("update site set site_pull = '%s' where site_url = '%s'", - dbesc(datetime_convert()), - dbesc($url) - ); + if ($blacklisted) { + if ($attempts > 20) + break; + $attempts--; + continue; + } - $j = json_decode($x['body'], true); - if ($j['success'] && $j['messages']) { - $sys = get_sys_channel(); - foreach ($j['messages'] as $message) { - // on these posts, clear any route info. - $message['route'] = ''; - process_delivery(['hash' => 'undefined'], get_item_elements($message), - [['hash' => $sys['xchan_hash']]], false, true); - $total++; + if ($url) { + if ($r[0]['site_pull'] > NULL_DATE) + $mindate = urlencode(datetime_convert('', '', $r[0]['site_pull'] . ' - 1 day')); + else { + $days = get_config('externals', 'since_days'); + if ($days === false) + $days = 15; + $mindate = urlencode(datetime_convert('', '', 'now - ' . intval($days) . ' days')); + } + + $feedurl = $url . '/zotfeed?f=&mindate=' . $mindate; + + logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG); + + $x = z_fetch_url($feedurl); + if (($x) && ($x['success'])) { + + q("update site set site_pull = '%s' where site_url = '%s'", + dbesc(datetime_convert()), + dbesc($url) + ); + + $j = json_decode($x['body'], true); + if ($j['success'] && $j['messages']) { + $sys = get_sys_channel(); + foreach ($j['messages'] as $message) { + // on these posts, clear any route info. + $message['route'] = ''; + process_delivery(['hash' => 'undefined'], get_item_elements($message), + [['hash' => $sys['xchan_hash']]], false, true); + $total++; + } + logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG); + } } - logger('externals: import_public_posts: ' . $total . ' messages imported', LOGGER_DEBUG); } - } - } - } + }*/ } } -- cgit v1.2.3