From ef67e181610d6a116b90b151cdd363183c674f11 Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 18 Jan 2021 14:29:01 +0000 Subject: more cleanup daemon/onepoll --- Zotlabs/Daemon/Onepoll.php | 109 ++++++++++++++++++++++----------------------- 1 file changed, 53 insertions(+), 56 deletions(-) (limited to 'Zotlabs/Daemon/Onepoll.php') diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index 93a5412b0..0228f10fb 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -2,6 +2,7 @@ namespace Zotlabs\Daemon; +use App; use Zotlabs\Lib\Libzot; require_once('include/zot.php'); @@ -10,20 +11,18 @@ require_once('include/socgraph.php'); class Onepoll { - static public function run($argc,$argv) { + static public function run($argc, $argv) { logger('onepoll: start'); - - if(($argc > 1) && (intval($argv[1]))) + + if (($argc > 1) && (intval($argv[1]))) $contact_id = intval($argv[1]); - if(! $contact_id) { + if (!$contact_id) { logger('onepoll: no contact'); return; } - $d = datetime_convert(); - $contacts = q("SELECT abook.*, xchan.*, account.* FROM abook LEFT JOIN account on abook_account = account_id left join xchan on xchan_hash = abook_xchan where abook_id = %d @@ -32,38 +31,36 @@ class Onepoll { intval($contact_id), intval(ACCOUNT_OK), intval(ACCOUNT_UNVERIFIED) - ); + ); - if(! $contacts) { + if (!$contacts) { logger('onepoll: abook_id not found: ' . $contact_id); return; } $contact = $contacts[0]; - $t = $contact['abook_updated']; - $importer_uid = $contact['abook_channel']; - + $r = q("SELECT * from channel left join xchan on channel_hash = xchan_hash where channel_id = %d limit 1", intval($importer_uid) ); - if(! $r) + if (!$r) return; $importer = $r[0]; logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['xchan_name']}, CONTACT: {$contact['xchan_name']}"); - $last_update = ((($contact['abook_updated'] === $contact['abook_created']) || ($contact['abook_updated'] <= NULL_DATE)) - ? datetime_convert('UTC','UTC','now - 7 days') - : datetime_convert('UTC','UTC',$contact['abook_updated'] . ' - 2 days') + $last_update = ((($contact['abook_updated'] === $contact['abook_created']) || ($contact['abook_updated'] <= NULL_DATE)) + ? datetime_convert('UTC', 'UTC', 'now - 7 days') + : datetime_convert('UTC', 'UTC', $contact['abook_updated'] . ' - 2 days') ); - if($contact['xchan_network'] === 'rss') { + if ($contact['xchan_network'] === 'rss') { logger('onepoll: processing feed ' . $contact['xchan_name'], LOGGER_DEBUG); - $alive = handle_feed($importer['channel_id'],$contact_id,$contact['xchan_hash']); + $alive = handle_feed($importer['channel_id'], $contact_id, $contact['xchan_hash']); if ($alive) { q("update abook set abook_connected = '%s' where abook_id = %d", dbesc(datetime_convert()), @@ -72,22 +69,22 @@ class Onepoll { } return; } - - if(! in_array($contact['xchan_network'],['zot','zot6'])) + + if (!in_array($contact['xchan_network'], ['zot', 'zot6'])) return; // update permissions - if($contact['xchan_network'] === 'zot6') - $x = Libzot::refresh($contact,$importer); + if ($contact['xchan_network'] === 'zot6') + $x = Libzot::refresh($contact, $importer); - if($contact['xchan_network'] === 'zot') - $x = zot_refresh($contact,$importer); + if ($contact['xchan_network'] === 'zot') + $x = zot_refresh($contact, $importer); $responded = false; $updated = datetime_convert(); $connected = datetime_convert(); - if(! $x) { + if (!$x) { // mark for death by not updating abook_connected, this is caught in include/poller.php q("update abook set abook_updated = '%s' where abook_id = %d", dbesc($updated), @@ -103,38 +100,38 @@ class Onepoll { $responded = true; } - if(! $responded) + if (!$responded) return; - if($contact['xchan_connurl']) { + if ($contact['xchan_connurl']) { $fetch_feed = true; - $x = null; + $x = null; // They haven't given us permission to see their stream - $can_view_stream = intval(get_abconfig($importer_uid,$contact['abook_xchan'],'their_perms','view_stream')); + $can_view_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'their_perms', 'view_stream')); - if(! $can_view_stream) + if (!$can_view_stream) $fetch_feed = false; // we haven't given them permission to send us their stream - $can_send_stream = intval(get_abconfig($importer_uid,$contact['abook_xchan'],'my_perms','send_stream')); - - if(! $can_send_stream) + $can_send_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'my_perms', 'send_stream')); + + if (!$can_send_stream) $fetch_feed = false; - if($fetch_feed) { + if ($fetch_feed) { - if(strpos($contact['xchan_connurl'],z_root()) === 0) { + if (strpos($contact['xchan_connurl'], z_root()) === 0) { // local channel - save a network fetch $c = channelx_by_hash($contact['xchan_hash']); - if($c) { - $x = [ - 'success' => true, - 'body' => json_encode( [ - 'success' => true, - 'messages' => zot_feed($c['channel_id'], $importer['xchan_hash'], [ 'mindate' => $last_update ]) + if ($c) { + $x = [ + 'success' => true, + 'body' => json_encode([ + 'success' => true, + 'messages' => zot_feed($c['channel_id'], $importer['xchan_hash'], ['mindate' => $last_update]) ]) ]; } @@ -142,43 +139,43 @@ class Onepoll { else { // remote fetch - $feedurl = str_replace('/poco/','/zotfeed/',$contact['xchan_connurl']); - $feedurl .= '?f=&mindate=' . urlencode($last_update) . '&zid=' . $importer['channel_address'] . '@' . \App::get_hostname(); + $feedurl = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']); + $feedurl .= '?f=&mindate=' . urlencode($last_update) . '&zid=' . $importer['channel_address'] . '@' . App::get_hostname(); $recurse = 0; - $x = z_fetch_url($feedurl, false, $recurse, [ 'session' => true ]); + $x = z_fetch_url($feedurl, false, $recurse, ['session' => true]); } - logger('feed_update: ' . print_r($x,true), LOGGER_DATA); + logger('feed_update: ' . print_r($x, true), LOGGER_DATA); } - if(($x) && ($x['success'])) { + if (($x) && ($x['success'])) { $total = 0; logger('onepoll: feed update ' . $contact['xchan_name'] . ' ' . $feedurl); - $j = json_decode($x['body'],true); - if($j['success'] && $j['messages']) { - foreach($j['messages'] as $message) { - $results = process_delivery(array('hash' => $contact['xchan_hash']), get_item_elements($message), - array(array('hash' => $importer['xchan_hash'])), false); - logger('onepoll: feed_update: process_delivery: ' . print_r($results,true), LOGGER_DATA); - $total ++; + $j = json_decode($x['body'], true); + if ($j['success'] && $j['messages']) { + foreach ($j['messages'] as $message) { + $results = process_delivery(['hash' => $contact['xchan_hash']], get_item_elements($message), + [['hash' => $importer['xchan_hash']]], false); + logger('onepoll: feed_update: process_delivery: ' . print_r($results, true), LOGGER_DATA); + $total++; } logger("onepoll: $total messages processed"); } } } - + // update the poco details for this connection - if($contact['xchan_connurl']) { + if ($contact['xchan_connurl']) { $r = q("SELECT xlink_id from xlink where xlink_xchan = '%s' and xlink_updated > %s - INTERVAL %s and xlink_static = 0 limit 1", intval($contact['xchan_hash']), db_utcnow(), db_quoteinterval('1 DAY') ); - if(! $r) { - poco_load($contact['xchan_hash'],$contact['xchan_connurl']); + if (!$r) { + poco_load($contact['xchan_hash'], $contact['xchan_connurl']); } } -- cgit v1.2.3 From b242347fa1d41695edcdc378a2403b1621303da2 Mon Sep 17 00:00:00 2001 From: Mario Date: Wed, 20 Jan 2021 20:17:50 +0000 Subject: onepoll via zot6 to /zotfeed which implements an outbox --- Zotlabs/Daemon/Onepoll.php | 148 +++++++++++++++++++++++++++------------------ 1 file changed, 88 insertions(+), 60 deletions(-) (limited to 'Zotlabs/Daemon/Onepoll.php') diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index 0228f10fb..568745608 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -2,13 +2,13 @@ namespace Zotlabs\Daemon; -use App; +use Zotlabs\Lib\Activity; +use Zotlabs\Lib\ActivityStreams; +use Zotlabs\Lib\ASCollection; use Zotlabs\Lib\Libzot; -require_once('include/zot.php'); require_once('include/socgraph.php'); - class Onepoll { static public function run($argc, $argv) { @@ -38,8 +38,7 @@ class Onepoll { return; } - $contact = $contacts[0]; - + $contact = array_shift($contacts); $importer_uid = $contact['abook_channel']; $r = q("SELECT * from channel left join xchan on channel_hash = xchan_hash where channel_id = %d limit 1", @@ -53,6 +52,7 @@ class Onepoll { logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['xchan_name']}, CONTACT: {$contact['xchan_name']}"); + // TODO: unused $last_update = ((($contact['abook_updated'] === $contact['abook_created']) || ($contact['abook_updated'] <= NULL_DATE)) ? datetime_convert('UTC', 'UTC', 'now - 7 days') : datetime_convert('UTC', 'UTC', $contact['abook_updated'] . ' - 2 days') @@ -103,80 +103,108 @@ class Onepoll { if (!$responded) return; - if ($contact['xchan_connurl']) { - $fetch_feed = true; - $x = null; + $fetch_feed = true; + $x = null; - // They haven't given us permission to see their stream + // They haven't given us permission to see their stream - $can_view_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'their_perms', 'view_stream')); + $can_view_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'their_perms', 'view_stream')); - if (!$can_view_stream) - $fetch_feed = false; + if (!$can_view_stream) + $fetch_feed = false; - // we haven't given them permission to send us their stream + // we haven't given them permission to send us their stream - $can_send_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'my_perms', 'send_stream')); + $can_send_stream = intval(get_abconfig($importer_uid, $contact['abook_xchan'], 'my_perms', 'send_stream')); - if (!$can_send_stream) - $fetch_feed = false; + if (!$can_send_stream) + $fetch_feed = false; - if ($fetch_feed) { + if ($fetch_feed) { - if (strpos($contact['xchan_connurl'], z_root()) === 0) { - // local channel - save a network fetch - $c = channelx_by_hash($contact['xchan_hash']); - if ($c) { - $x = [ - 'success' => true, - 'body' => json_encode([ - 'success' => true, - 'messages' => zot_feed($c['channel_id'], $importer['xchan_hash'], ['mindate' => $last_update]) - ]) - ]; - } + $max = intval(get_config('system', 'max_imported_posts', 30)); + + if (intval($max)) { + $cl = get_xconfig($contact['abook_xchan'], 'activitypub', 'collections'); + + if (is_array($cl) && $cl) { + $url = ((array_key_exists('outbox', $cl)) ? $cl['outbox'] : ''); } else { - // remote fetch - - $feedurl = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']); - $feedurl .= '?f=&mindate=' . urlencode($last_update) . '&zid=' . $importer['channel_address'] . '@' . App::get_hostname(); - $recurse = 0; - $x = z_fetch_url($feedurl, false, $recurse, ['session' => true]); + $url = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']); } - logger('feed_update: ' . print_r($x, true), LOGGER_DATA); - } - - if (($x) && ($x['success'])) { - $total = 0; - logger('onepoll: feed update ' . $contact['xchan_name'] . ' ' . $feedurl); - - $j = json_decode($x['body'], true); - if ($j['success'] && $j['messages']) { - foreach ($j['messages'] as $message) { - $results = process_delivery(['hash' => $contact['xchan_hash']], get_item_elements($message), - [['hash' => $importer['xchan_hash']]], false); - logger('onepoll: feed_update: process_delivery: ' . print_r($results, true), LOGGER_DATA); - $total++; + if ($url) { + logger('fetching outbox'); + $obj = new ASCollection($url, $importer, 0, $max); + $messages = $obj->get(); + if ($messages) { + foreach ($messages as $message) { + if (is_string($message)) { + $message = Activity::fetch($message, $importer); + } + $AS = new ActivityStreams($message); + if ($AS->is_valid() && is_array($AS->obj)) { + $item = Activity::decode_note($AS); + Activity::store($importer, $contact['abook_xchan'], $AS, $item); + } + } } - logger("onepoll: $total messages processed"); } } } + /* if ($fetch_feed) { + + if (strpos($contact['xchan_connurl'], z_root()) === 0) { + // local channel - save a network fetch + $c = channelx_by_hash($contact['xchan_hash']); + if ($c) { + $x = [ + 'success' => true, + 'body' => json_encode([ + 'success' => true, + 'messages' => zot_feed($c['channel_id'], $importer['xchan_hash'], ['mindate' => $last_update]) + ]) + ]; + } + } + else { + // remote fetch + + $feedurl = str_replace('/poco/', '/zotfeed/', $contact['xchan_connurl']); + $feedurl .= '?f=&mindate=' . urlencode($last_update) . '&zid=' . $importer['channel_address'] . '@' . App::get_hostname(); + $recurse = 0; + $x = z_fetch_url($feedurl, false, $recurse, ['session' => true]); + } + + logger('feed_update: ' . print_r($x, true), LOGGER_DATA); + } - // update the poco details for this connection + if (($x) && ($x['success'])) { + $total = 0; + logger('onepoll: feed update ' . $contact['xchan_name'] . ' ' . $feedurl); + + $j = json_decode($x['body'], true); + if ($j['success'] && $j['messages']) { + foreach ($j['messages'] as $message) { + $results = process_delivery(['hash' => $contact['xchan_hash']], get_item_elements($message), + [['hash' => $importer['xchan_hash']]], false); + logger('onepoll: feed_update: process_delivery: ' . print_r($results, true), LOGGER_DATA); + $total++; + } + logger("onepoll: $total messages processed"); + } + } + */ - if ($contact['xchan_connurl']) { - $r = q("SELECT xlink_id from xlink - where xlink_xchan = '%s' and xlink_updated > %s - INTERVAL %s and xlink_static = 0 limit 1", - intval($contact['xchan_hash']), - db_utcnow(), db_quoteinterval('1 DAY') - ); - if (!$r) { - poco_load($contact['xchan_hash'], $contact['xchan_connurl']); - } + // update the poco details for this connection + $r = q("SELECT xlink_id from xlink where xlink_xchan = '%s' and xlink_updated > %s - INTERVAL %s and xlink_static = 0 limit 1", + intval($contact['xchan_hash']), + db_utcnow(), db_quoteinterval('1 DAY') + ); + if (!$r) { + poco_load($contact['xchan_hash'], $contact['xchan_connurl']); } return; -- cgit v1.2.3 From 57645386b2cd0adb1e8aa7a69d638853b32b6f08 Mon Sep 17 00:00:00 2001 From: Mario Date: Wed, 20 Jan 2021 20:42:59 +0000 Subject: looks like we get arrays or json strings here for some reason --- Zotlabs/Daemon/Onepoll.php | 1 + 1 file changed, 1 insertion(+) (limited to 'Zotlabs/Daemon/Onepoll.php') diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index 568745608..388a78c81 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -138,6 +138,7 @@ class Onepoll { logger('fetching outbox'); $obj = new ASCollection($url, $importer, 0, $max); $messages = $obj->get(); + hz_syslog(print_r($messages,true)); if ($messages) { foreach ($messages as $message) { if (is_string($message)) { -- cgit v1.2.3 From c5d37a08314c27afeeada724872e28ee7632a080 Mon Sep 17 00:00:00 2001 From: Mario Date: Wed, 20 Jan 2021 20:43:51 +0000 Subject: remove logging --- Zotlabs/Daemon/Onepoll.php | 1 - 1 file changed, 1 deletion(-) (limited to 'Zotlabs/Daemon/Onepoll.php') diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index 388a78c81..568745608 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -138,7 +138,6 @@ class Onepoll { logger('fetching outbox'); $obj = new ASCollection($url, $importer, 0, $max); $messages = $obj->get(); - hz_syslog(print_r($messages,true)); if ($messages) { foreach ($messages as $message) { if (is_string($message)) { -- cgit v1.2.3 From 0ffb09aa40d4bb2e05dd3b1ae306cb924f86ea12 Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 11 Feb 2021 21:05:45 +0000 Subject: onepoll: add date_begin argument to poll url --- Zotlabs/Daemon/Onepoll.php | 1 + 1 file changed, 1 insertion(+) (limited to 'Zotlabs/Daemon/Onepoll.php') diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index 568745608..2200276e0 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -136,6 +136,7 @@ class Onepoll { if ($url) { logger('fetching outbox'); + $url = $url . '?date_begin=' . urlencode($last_update); $obj = new ASCollection($url, $importer, 0, $max); $messages = $obj->get(); if ($messages) { -- cgit v1.2.3 From c3ec5d4d6ac48988b70d17b0b08b35da92cfa24f Mon Sep 17 00:00:00 2001 From: Mario Date: Wed, 17 Feb 2021 20:16:20 +0000 Subject: do not try to fetch legacy zot zotfeed - they will not return anything useful --- Zotlabs/Daemon/Onepoll.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'Zotlabs/Daemon/Onepoll.php') diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index 2200276e0..85394c8d0 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -24,7 +24,7 @@ class Onepoll { } $contacts = q("SELECT abook.*, xchan.*, account.* - FROM abook LEFT JOIN account on abook_account = account_id left join xchan on xchan_hash = abook_xchan + FROM abook LEFT JOIN account on abook_account = account_id left join xchan on xchan_hash = abook_xchan where abook_id = %d and abook_pending = 0 and abook_archived = 0 and abook_blocked = 0 and abook_ignored = 0 AND (( account_flags = %d ) OR ( account_flags = %d )) limit 1", @@ -52,7 +52,6 @@ class Onepoll { logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['xchan_name']}, CONTACT: {$contact['xchan_name']}"); - // TODO: unused $last_update = ((($contact['abook_updated'] === $contact['abook_created']) || ($contact['abook_updated'] <= NULL_DATE)) ? datetime_convert('UTC', 'UTC', 'now - 7 days') : datetime_convert('UTC', 'UTC', $contact['abook_updated'] . ' - 2 days') @@ -120,7 +119,7 @@ class Onepoll { if (!$can_send_stream) $fetch_feed = false; - if ($fetch_feed) { + if ($fetch_feed && $contact['xchan_network'] !== 'zot') { $max = intval(get_config('system', 'max_imported_posts', 30)); -- cgit v1.2.3 From 373612a0465c924f5365d0e8d3a505da04b7a99f Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 25 Feb 2021 15:05:49 +0000 Subject: do not poll feeds if feed_contacts are not allowed --- Zotlabs/Daemon/Onepoll.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'Zotlabs/Daemon/Onepoll.php') diff --git a/Zotlabs/Daemon/Onepoll.php b/Zotlabs/Daemon/Onepoll.php index 85394c8d0..598cf28e4 100644 --- a/Zotlabs/Daemon/Onepoll.php +++ b/Zotlabs/Daemon/Onepoll.php @@ -23,9 +23,15 @@ class Onepoll { return; } + $sql_extra = ''; + $allow_feeds = get_config('system', 'feed_contacts'); + if(!$allow_feeds) { + $sql_extra = ' and abook_feed = 0 '; + } + $contacts = q("SELECT abook.*, xchan.*, account.* FROM abook LEFT JOIN account on abook_account = account_id left join xchan on xchan_hash = abook_xchan - where abook_id = %d + where abook_id = %d $sql_extra and abook_pending = 0 and abook_archived = 0 and abook_blocked = 0 and abook_ignored = 0 AND (( account_flags = %d ) OR ( account_flags = %d )) limit 1", intval($contact_id), -- cgit v1.2.3