From 0dcbcf93c21a9e143aa580a836403c634429a90c Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 12 Nov 2020 11:50:00 +0000 Subject: some work on include/import --- include/import.php | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/include/import.php b/include/import.php index 910cb8be7..82c23e116 100644 --- a/include/import.php +++ b/include/import.php @@ -210,17 +210,24 @@ function import_hublocs($channel, $hublocs, $seize, $moving = false) { if($channel && $hublocs) { foreach($hublocs as $hubloc) { - $hash = make_xchan_hash($hubloc['hubloc_guid'],$hubloc['hubloc_guid_sig']); - if($hubloc['hubloc_network'] === 'zot' && $hash !== $hubloc['hubloc_hash']) { - logger('forged hubloc: ' . print_r($hubloc,true)); - continue; - } + // verify the hash. We can only do this if we already stored the xchan corresponding to this hubloc + // as we need the public key from there - if(! array_key_exists('hubloc_primary',$hubloc)) { - $hubloc['hubloc_primary'] = (($hubloc['hubloc_flags'] & 0x0001) ? 1 : 0); - $hubloc['hubloc_orphancheck'] = (($hubloc['hubloc_flags'] & 0x0004) ? 1 : 0); - $hubloc['hubloc_error'] = (($hubloc['hubloc_status'] & 0x0003) ? 1 : 0); - $hubloc['hubloc_deleted'] = (($hubloc['hubloc_flags'] & 0x1000) ? 1 : 0); + if ($hubloc['hubloc_network'] === 'zot6') { + $x = q("select xchan_pubkey from xchan where xchan_guid = '%s' and xchan_hash = '%s'", + dbesc($hubloc['hubloc_guid']), + dbesc($hubloc['hubloc_hash']) + ); + + if (! $x) { + logger('hubloc could not be verified. ' . print_r($hubloc,true)); + continue; + } + $hash = Libzot::make_xchan_hash($hubloc['hubloc_guid'],$x[0]['xchan_pubkey']); + if ($hash !== $hubloc['hubloc_hash']) { + logger('forged hubloc: ' . print_r($hubloc,true)); + continue; + } } if($moving && $hubloc['hubloc_hash'] === $channel['channel_hash'] && $hubloc['hubloc_url'] !== z_root()) { @@ -228,17 +235,17 @@ function import_hublocs($channel, $hublocs, $seize, $moving = false) { } $arr = [ - 'guid' => $hubloc['hubloc_guid'], - 'guid_sig' => $hubloc['hubloc_guid_sig'], - 'url' => $hubloc['hubloc_url'], - 'url_sig' => $hubloc['hubloc_url_sig'], - 'sitekey' => ((array_key_exists('hubloc_sitekey',$hubloc)) ? $hubloc['hubloc_sitekey'] : '') + 'id' => $hubloc['hubloc_guid'], + 'id_sig' => $hubloc['hubloc_guid_sig'], + 'location' => $hubloc['hubloc_url'], + 'location_sig' => $hubloc['hubloc_url_sig'] ]; - if(($hubloc['hubloc_hash'] === $channel['channel_hash']) && intval($hubloc['hubloc_primary']) && ($seize)) + if (($hubloc['hubloc_hash'] === $channel['channel_hash']) && intval($hubloc['hubloc_primary']) && ($seize)) { $hubloc['hubloc_primary'] = 0; + } - if(($x = zot_gethub($arr,false)) === false) { + if (($x = Libzot::gethub($arr,false)) === false) { unset($hubloc['hubloc_id']); hubloc_store_lowlevel($hubloc); } @@ -1333,7 +1340,7 @@ function sync_files($channel, $files) { 'time' => $time, 'resource' => $att['hash'], 'revision' => 0, - 'signature' => base64url_encode(rsa_sign($channel['channel_hash'] . '.' . $time, $channel['channel_prvkey'])) + 'signature' => Libzot::sign($channel['channel_hash'] . '.' . $time, $channel['channel_prvkey']) ); $store_path = $newfname; @@ -1419,7 +1426,7 @@ function sync_files($channel, $files) { 'time' => $time, 'resource' => $p['resource_id'], 'revision' => 0, - 'signature' => base64url_encode(rsa_sign($channel['channel_hash'] . '.' . $time, $channel['channel_prvkey'])), + 'signature' => Libzot::sign($channel['channel_hash'] . '.' . $time, $channel['channel_prvkey']), 'resolution' => intval($p['imgscale']) ); -- cgit v1.2.3 From a8de999b7c87d1cd05f625f0d950694f60e28f24 Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 12 Nov 2020 20:17:03 +0000 Subject: do not store actors without an inbox - they are not valid --- Zotlabs/Lib/Activity.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php index 48714a48d..a8e0da776 100644 --- a/Zotlabs/Lib/Activity.php +++ b/Zotlabs/Lib/Activity.php @@ -1542,6 +1542,12 @@ class Activity { $inbox = $person_obj['inbox']; + // invalid identity + + if (! $inbox) { + return; + } + $collections = []; if($inbox) { -- cgit v1.2.3 From 16071f300ee197bb8ef460e714e7eb5936622596 Mon Sep 17 00:00:00 2001 From: Mario Date: Fri, 13 Nov 2020 09:18:54 +0000 Subject: better check for valid inbox --- Zotlabs/Lib/Activity.php | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php index a8e0da776..5f222754b 100644 --- a/Zotlabs/Lib/Activity.php +++ b/Zotlabs/Lib/Activity.php @@ -1493,6 +1493,14 @@ class Activity { if(! is_array($person_obj)) return; + $inbox = $person_obj['inbox']; + + // invalid identity + + if (! $inbox || strpos($inbox,z_root()) !== false) { + return; + } + $name = $person_obj['name']; if(! $name) $name = $person_obj['preferredUsername']; @@ -1540,14 +1548,6 @@ class Activity { $profile = $url; } - $inbox = $person_obj['inbox']; - - // invalid identity - - if (! $inbox) { - return; - } - $collections = []; if($inbox) { -- cgit v1.2.3 From b6e34695197191275d6d9f4f63f7f976bcf89a2d Mon Sep 17 00:00:00 2001 From: Mario Date: Fri, 13 Nov 2020 09:26:04 +0000 Subject: update to fix more broken hublocs and version bump --- Zotlabs/Update/_1239.php | 27 +++++++++++++++++++++++++++ boot.php | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 Zotlabs/Update/_1239.php diff --git a/Zotlabs/Update/_1239.php b/Zotlabs/Update/_1239.php new file mode 100644 index 000000000..ed8de30bb --- /dev/null +++ b/Zotlabs/Update/_1239.php @@ -0,0 +1,27 @@ + Date: Fri, 13 Nov 2020 13:49:39 +0000 Subject: minor queue adjustments --- Zotlabs/Daemon/Notifier.php | 23 +++++++++++++---------- include/network.php | 5 +++-- include/zot.php | 4 ++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Zotlabs/Daemon/Notifier.php b/Zotlabs/Daemon/Notifier.php index 8ea75af61..f24eab773 100644 --- a/Zotlabs/Daemon/Notifier.php +++ b/Zotlabs/Daemon/Notifier.php @@ -4,6 +4,7 @@ namespace Zotlabs\Daemon; use Zotlabs\Lib\Libzot; use Zotlabs\Lib\Activity; +use Zotlabs\Lib\Queue; require_once('include/queue_fn.php'); require_once('include/html2plain.php'); @@ -734,15 +735,17 @@ class Notifier { } if($packet) { - queue_insert(array( - 'hash' => $hash, - 'account_id' => $channel['channel_account_id'], - 'channel_id' => $channel['channel_id'], - 'posturl' => $hub['hubloc_callback'], - 'driver' => $hub['hubloc_network'], - 'notify' => $packet, - 'msg' => (($pmsg) ? json_encode($pmsg) : '') - )); + Queue::insert( + [ + 'hash' => $hash, + 'account_id' => $channel['channel_account_id'], + 'channel_id' => $channel['channel_id'], + 'posturl' => $hub['hubloc_callback'], + 'driver' => $hub['hubloc_network'], + 'notify' => $packet, + 'msg' => (($pmsg) ? json_encode($pmsg) : '') + ] + ); } else { $env = (($hub_env && $hub_env[$hub['hubloc_host'] . $hub['hubloc_sitekey']]) ? $hub_env[$hub['hubloc_host'] . $hub['hubloc_sitekey']] : ''); @@ -803,7 +806,7 @@ class Notifier { } } - queue_insert( + Queue::insert( [ 'hash' => $hash, 'account_id' => $target_item['aid'], diff --git a/include/network.php b/include/network.php index 9d4c00ee8..bcd66cf85 100644 --- a/include/network.php +++ b/include/network.php @@ -2,6 +2,7 @@ use Zotlabs\Lib\Zotfinger; use Zotlabs\Lib\Libzot; +use Zotlabs\Lib\Queue; /** * @file include/network.php @@ -1437,10 +1438,10 @@ function do_delivery($deliveries, $force = false) { $x = q("select count(outq_hash) as total from outq where outq_delivered = 0"); - if(intval($x[0]['total']) > intval(get_config('system','force_queue_threshold',300)) && (! $force)) { + if(intval($x[0]['total']) > intval(get_config('system','force_queue_threshold',3000)) && (! $force)) { logger('immediate delivery deferred.', LOGGER_DEBUG, LOG_INFO); foreach($deliveries as $d) { - update_queue_item($d); + Queue::update($d); } return; } diff --git a/include/zot.php b/include/zot.php index 8a2177de7..d61873ba2 100644 --- a/include/zot.php +++ b/include/zot.php @@ -3452,7 +3452,7 @@ function build_sync_packet($uid = 0, $packet = null, $groups_changed = false) { $x = q("select count(outq_hash) as total from outq where outq_delivered = 0"); - if(intval($x[0]['total']) > intval(get_config('system','force_queue_threshold',300))) { + if(intval($x[0]['total']) > intval(get_config('system','force_queue_threshold',3000))) { logger('immediate delivery deferred.', LOGGER_DEBUG, LOG_INFO); update_queue_item($hash); continue; @@ -4286,7 +4286,7 @@ function zot_reply_message_request($data) { $x = q("select count(outq_hash) as total from outq where outq_delivered = 0"); - if(intval($x[0]['total']) > intval(get_config('system','force_queue_threshold',300))) { + if(intval($x[0]['total']) > intval(get_config('system','force_queue_threshold',3000))) { logger('immediate delivery deferred.', LOGGER_DEBUG, LOG_INFO); update_queue_item($hash); continue; -- cgit v1.2.3 From b63c5f27853aded30c492bdb94a680e5a9c9648b Mon Sep 17 00:00:00 2001 From: Mario Date: Sat, 14 Nov 2020 21:28:50 +0000 Subject: Polling fallback to server sent events. Polling is the default. SSE must be enabled in /admin/site > Advanced > Enable SSE Notifications if desired. --- Zotlabs/Module/Admin/Site.php | 8 ++++ Zotlabs/Module/Sse.php | 107 +++++++++++++++++++++++++++++++----------- boot.php | 1 + view/js/main.js | 82 ++++++++++++++++++++++---------- view/tpl/admin_site.tpl | 1 + view/tpl/head.tpl | 1 + 6 files changed, 147 insertions(+), 53 deletions(-) diff --git a/Zotlabs/Module/Admin/Site.php b/Zotlabs/Module/Admin/Site.php index 4bb34b7b7..2c85268a7 100644 --- a/Zotlabs/Module/Admin/Site.php +++ b/Zotlabs/Module/Admin/Site.php @@ -62,6 +62,9 @@ class Site { $from_email = ((array_key_exists('from_email',$_POST) && trim($_POST['from_email'])) ? trim($_POST['from_email']) : 'Administrator@' . \App::get_hostname()); $from_email_name = ((array_key_exists('from_email_name',$_POST) && trim($_POST['from_email_name'])) ? trim($_POST['from_email_name']) : \Zotlabs\Lib\System::get_site_name()); + + $sse_enabled = ((x($_POST,'sse_enabled')) ? true : false); + $verifyssl = ((x($_POST,'verifyssl')) ? True : False); $proxyuser = ((x($_POST,'proxyuser')) ? notags(trim($_POST['proxyuser'])) : ''); $proxy = ((x($_POST,'proxy')) ? notags(trim($_POST['proxy'])) : ''); @@ -151,6 +154,9 @@ class Site { set_config('system','no_community_page', $no_community_page); set_config('system','no_utf', $no_utf); + + set_config('system','sse_enabled', $sse_enabled); + set_config('system','verifyssl', $verifyssl); set_config('system','proxyuser', $proxyuser); set_config('system','proxy', $proxy); @@ -331,6 +337,8 @@ class Site { '$directory_server' => (($dir_choices) ? array('directory_server', t("Directory Server URL"), get_config('system','directory_server'), t("Default directory server"), $dir_choices) : null), + '$sse_enabled' => array('sse_enabled', t('Enable SSE Notifications'), get_config('system', 'sse_enabled', 0), t('If disabled, traditional polling will be used. Warning: this setting might not be suited for shared hosting.')), + '$proxyuser' => array('proxyuser', t("Proxy user"), get_config('system','proxyuser'), ""), '$proxy' => array('proxy', t("Proxy URL"), get_config('system','proxy'), ""), '$timeout' => array('timeout', t("Network timeout"), (x(get_config('system','curl_timeout'))?get_config('system','curl_timeout'):60), t("Value is in seconds. Set to 0 for unlimited (not recommended).")), diff --git a/Zotlabs/Module/Sse.php b/Zotlabs/Module/Sse.php index b68fe6705..46b4a8d87 100644 --- a/Zotlabs/Module/Sse.php +++ b/Zotlabs/Module/Sse.php @@ -14,6 +14,7 @@ class Sse extends Controller { public static $ob_hash; public static $sse_id; public static $vnotify; + public static $sse_enabled; function init() { @@ -49,18 +50,86 @@ class Sse extends Controller { $sys = get_sys_channel(); $sleep_seconds = 3; - header("Content-Type: text/event-stream"); - header("Cache-Control: no-cache"); - header("Connection: keep-alive"); - header("X-Accel-Buffering: no"); + self::$sse_enabled = get_config('system', 'sse_enabled', 0); - while(true) { + if(self::$sse_enabled) { - /** - * Update chat presence indication (if applicable) - */ + // Server Sent Events + + header("Content-Type: text/event-stream"); + header("Cache-Control: no-cache"); + header("Connection: keep-alive"); + header("X-Accel-Buffering: no"); + + while(true) { + + if(! self::$sse_id) { + + // Update chat presence indication + + $r = q("select cp_id, cp_room from chatpresence where cp_xchan = '%s' and cp_client = '%s' and cp_room = 0 limit 1", + dbesc(self::$ob_hash), + dbesc($_SERVER['REMOTE_ADDR']) + ); + $basic_presence = false; + if($r) { + $basic_presence = true; + q("update chatpresence set cp_last = '%s' where cp_id = %d", + dbesc(datetime_convert()), + intval($r[0]['cp_id']) + ); + } + if(! $basic_presence) { + q("insert into chatpresence ( cp_xchan, cp_last, cp_status, cp_client) + values( '%s', '%s', '%s', '%s' ) ", + dbesc(self::$ob_hash), + dbesc(datetime_convert()), + dbesc('online'), + dbesc($_SERVER['REMOTE_ADDR']) + ); + } + } + + XConfig::Load(self::$ob_hash); + + $result = XConfig::Get(self::$ob_hash, 'sse', 'notifications', []); + $lock = XConfig::Get(self::$ob_hash, 'sse', 'lock'); + + if($result && !$lock) { + echo "event: notifications\n"; + echo 'data: ' . json_encode($result); + echo "\n\n"; + + XConfig::Set(self::$ob_hash, 'sse', 'notifications', []); + unset($result); + } + + // always send heartbeat to detect disconnected clients + echo "event: heartbeat\n"; + echo 'data: {}'; + echo "\n\n"; + + ob_end_flush(); + flush(); + + if(connection_status() != CONNECTION_NORMAL || connection_aborted()) { + //TODO: this does not seem to be triggered + XConfig::Set(self::$ob_hash, 'sse', 'timestamp', NULL_DATE); + break; + } + + sleep($sleep_seconds); + + } + + } + else { + // Fallback to traditional polling if(! self::$sse_id) { + + // Update chat presence indication + $r = q("select cp_id, cp_room from chatpresence where cp_xchan = '%s' and cp_client = '%s' and cp_room = 0 limit 1", dbesc(self::$ob_hash), dbesc($_SERVER['REMOTE_ADDR']) @@ -90,29 +159,11 @@ class Sse extends Controller { $lock = XConfig::Get(self::$ob_hash, 'sse', 'lock'); if($result && !$lock) { - echo "event: notifications\n"; - echo 'data: ' . json_encode($result); - echo "\n\n"; - XConfig::Set(self::$ob_hash, 'sse', 'notifications', []); - unset($result); - } - - // always send heartbeat to detect disconnected clients - echo "event: heartbeat\n"; - echo 'data: {}'; - echo "\n\n"; - - ob_end_flush(); - flush(); - - if(connection_status() != CONNECTION_NORMAL || connection_aborted()) { - //TODO: this does not seem to be triggered - XConfig::Set(self::$ob_hash, 'sse', 'timestamp', NULL_DATE); - break; + json_return_and_die($result); } - sleep($sleep_seconds); + killme(); } diff --git a/boot.php b/boot.php index 52761431b..f5c75259f 100755 --- a/boot.php +++ b/boot.php @@ -1207,6 +1207,7 @@ class App { '$metas' => self::$meta->get(), '$plugins' => $x['header'], '$update_interval' => $interval, + '$sse_enabled' => get_config('system', 'sse_enabled', 0), '$head_css' => head_get_css(), '$head_js' => head_get_js(), '$linkrel' => head_get_links(), diff --git a/view/js/main.js b/view/js/main.js index 8ceb0b143..560868046 100644 --- a/view/js/main.js +++ b/view/js/main.js @@ -31,6 +31,7 @@ var sse_offset = 0; var sse_type; var sse_partial_result = false; var sse_rmids = []; +var sse_fallback_interval; var page_cache = {}; @@ -94,37 +95,56 @@ $(document).ready(function() { jQuery.timeago.settings.allowFuture = true; - if(typeof(window.SharedWorker) === 'undefined') { - // notifications with multiple tabs open will not work very well in this scenario - var evtSource = new EventSource('/sse'); - evtSource.addEventListener('notifications', function(e) { - var obj = JSON.parse(e.data); - sse_handleNotifications(obj, false, false); - }, false); + if(sse_enabled) { + if(typeof(window.SharedWorker) === 'undefined') { + // notifications with multiple tabs open will not work very well in this scenario + var evtSource = new EventSource('/sse'); - document.addEventListener('visibilitychange', function() { - if (!document.hidden) { - sse_offset = 0; - sse_bs_init(); - } - }, false); + evtSource.addEventListener('notifications', function(e) { + var obj = JSON.parse(e.data); + sse_handleNotifications(obj, false, false); + }, false); - } - else { - var myWorker = new SharedWorker('/view/js/sse_worker.js', localUser); + document.addEventListener('visibilitychange', function() { + if (!document.hidden) { + sse_offset = 0; + sse_bs_init(); + } + }, false); - myWorker.port.onmessage = function(e) { - obj = e.data; - console.log(obj); - sse_handleNotifications(obj, false, false); } + else { + var myWorker = new SharedWorker('/view/js/sse_worker.js', localUser); - myWorker.onerror = function(e) { - myWorker.port.close(); + myWorker.port.onmessage = function(e) { + obj = e.data; + console.log(obj); + sse_handleNotifications(obj, false, false); + } + + myWorker.onerror = function(e) { + myWorker.port.close(); + } + + myWorker.port.start(); } + } + else { + if (!document.hidden) + sse_fallback_interval = setInterval(sse_fallback, updateInterval); - myWorker.port.start(); + document.addEventListener('visibilitychange', function() { + if (document.hidden) { + clearInterval(sse_fallback_interval); + } + else { + sse_offset = 0; + sse_bs_init(); + sse_fallback_interval = setInterval(sse_fallback, updateInterval); + } + + }, false); } $('.notification-link').on('click', { replace: true, followup: false }, sse_bs_notifications); @@ -224,6 +244,8 @@ $(document).ready(function() { cache_next_page(); }); + + }); function getConversationSettings() { @@ -1763,8 +1785,6 @@ function sse_bs_init() { } function sse_bs_counts() { - - if(sse_bs_active) return; @@ -2022,3 +2042,15 @@ function sse_setNotificationsStatus() { } } + +function sse_fallback() { + $.get('/sse', function(obj) { + if(! obj) + return; + + console.log('sse fallback'); + console.log(obj); + + sse_handleNotifications(obj, false, false); + }); +} diff --git a/view/tpl/admin_site.tpl b/view/tpl/admin_site.tpl index 5e10e6eea..8d32ba9a4 100755 --- a/view/tpl/admin_site.tpl +++ b/view/tpl/admin_site.tpl @@ -95,6 +95,7 @@

{{$advanced}}

+ {{include file="field_checkbox.tpl" field=$sse_enabled}} {{include file="field_input.tpl" field=$imagick_path}} {{include file="field_input.tpl" field=$proxy}} {{include file="field_input.tpl" field=$proxyuser}} diff --git a/view/tpl/head.tpl b/view/tpl/head.tpl index bd4cf3747..0d212e029 100755 --- a/view/tpl/head.tpl +++ b/view/tpl/head.tpl @@ -9,6 +9,7 @@ {{$plugins}}