From 7ce4175876ce13727432dc839938bb8e486ef406 Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Wed, 12 Apr 2023 09:29:51 +0200 Subject: initial commit directory sync rewrite - should work but still requires lots of cleanup --- Zotlabs/Daemon/Cron.php | 12 ++++++++ Zotlabs/Daemon/Onedirsync.php | 19 ++++++------ Zotlabs/Daemon/Poller.php | 9 ++++++ Zotlabs/Lib/Libsync.php | 2 +- Zotlabs/Lib/Libzot.php | 16 +++++------ Zotlabs/Lib/Libzotdir.php | 67 ++++++++++++++++++++++++++++++++++++------- Zotlabs/Module/Dirsearch.php | 5 +++- 7 files changed, 101 insertions(+), 29 deletions(-) diff --git a/Zotlabs/Daemon/Cron.php b/Zotlabs/Daemon/Cron.php index 640f06102..caae0ce53 100644 --- a/Zotlabs/Daemon/Cron.php +++ b/Zotlabs/Daemon/Cron.php @@ -3,6 +3,7 @@ namespace Zotlabs\Daemon; use Zotlabs\Lib\Libsync; +use Zotlabs\Lib\Libzotdir; class Cron { @@ -35,6 +36,17 @@ class Cron { logger('cron: start'); + // If this is a directory server, request a sync with an upstream + // directory at least once a day, up to once every poll interval. + // Pull remote changes and push local changes. + // potential issue: how do we keep from creating an endless update loop? + + $dirmode = get_config('system', 'directory_mode'); + + if ($dirmode == DIRECTORY_MODE_SECONDARY || $dirmode == DIRECTORY_MODE_PRIMARY) { + Libzotdir::sync_directories($dirmode); + } + // run queue delivery process in the background Master::Summon(array('Queue')); diff --git a/Zotlabs/Daemon/Onedirsync.php b/Zotlabs/Daemon/Onedirsync.php index ea995be9e..89f4a40ed 100644 --- a/Zotlabs/Daemon/Onedirsync.php +++ b/Zotlabs/Daemon/Onedirsync.php @@ -19,13 +19,14 @@ class Onedirsync { return; } - $r = q("select * from updates where ud_id = %d limit 1", + $r = q("select * from updates where ud_id = %d", intval($update_id) ); if (!$r) return; +/* if (($r[0]['ud_flags'] & UPDATE_FLAGS_UPDATED) || (!$r[0]['ud_addr'])) return; @@ -47,22 +48,24 @@ class Onedirsync { ); return; } - +*/ // ignore doing an update if this ud_addr refers to a known dead hubloc - $h = q("select * from hubloc where hubloc_addr = '%s'", + $h = q("select * from hubloc where hubloc_addr = '%s' order by hubloc_id desc", dbesc($r[0]['ud_addr']), ); $h = Libzot::zot_record_preferred($h); if (($h) && (($h['hubloc_status'] & HUBLOC_OFFLINE) || $h['hubloc_deleted'] || $h['hubloc_error'])) { - q("update updates set ud_flags = ( ud_flags | %d ) where ud_addr = '%s' and ( ud_flags & %d ) = 0 ", - intval(UPDATE_FLAGS_DELETED), - dbesc($r[0]['ud_addr']), - intval(UPDATE_FLAGS_UPDATED) + q("update updates set ud_flags = 9 where ud_hash = '%s' and ud_flags != 9", + dbesc($r[0]['ud_hash']) ); - return; + + // 2023-04-12: Flag the entry deleted but try to update anyway since the info is not always correct + // This might change after all directory servers run the new code. + + // return; } // we might have to pull this out some day, but for now update_directory_entry() diff --git a/Zotlabs/Daemon/Poller.php b/Zotlabs/Daemon/Poller.php index 0fdc3da16..537209530 100644 --- a/Zotlabs/Daemon/Poller.php +++ b/Zotlabs/Daemon/Poller.php @@ -183,11 +183,20 @@ class Poller { $dirmode = intval(get_config('system', 'directory_mode')); if ($dirmode == DIRECTORY_MODE_SECONDARY || $dirmode == DIRECTORY_MODE_PRIMARY) { +/* $r = q("SELECT u.ud_addr, u.ud_id, u.ud_last FROM updates AS u INNER JOIN (SELECT ud_addr, max(ud_id) AS ud_id FROM updates WHERE ( ud_flags & %d ) = 0 AND ud_addr != '' AND ( ud_last <= '%s' OR ud_last > %s - INTERVAL %s ) GROUP BY ud_addr) AS s ON s.ud_id = u.ud_id ", intval(UPDATE_FLAGS_UPDATED), dbesc(NULL_DATE), db_utcnow(), db_quoteinterval('7 DAY') ); +*/ + + $r = q("SELECT * FROM updates WHERE ud_flags = 1 AND (ud_last = '%s' OR ud_last > %s - INTERVAL %s)", + dbesc(NULL_DATE), + db_utcnow(), + db_quoteinterval('7 DAY') + ); + if ($r) { foreach ($r as $rr) { diff --git a/Zotlabs/Lib/Libsync.php b/Zotlabs/Lib/Libsync.php index d52b501e4..56bb411c4 100644 --- a/Zotlabs/Lib/Libsync.php +++ b/Zotlabs/Lib/Libsync.php @@ -786,7 +786,7 @@ class Libsync { if (isset($arr['locations']) && $arr['locations']) { - $xisting = q("select * from hubloc where hubloc_hash = '%s'", + $xisting = q("select * from hubloc where hubloc_hash = '%s' and hubloc_deleted = 0", dbesc($sender['hash']) ); diff --git a/Zotlabs/Lib/Libzot.php b/Zotlabs/Lib/Libzot.php index a8334595f..b41ba043a 100644 --- a/Zotlabs/Lib/Libzot.php +++ b/Zotlabs/Lib/Libzot.php @@ -920,11 +920,11 @@ class Libzot { $s = Libsync::sync_locations($arr, $arr); if ($s) { - if (isset($s['change_message'])) + if (!empty($s['change_message'])) $what .= $s['change_message']; - if (isset($s['changed'])) + if (!empty($s['changed'])) $changed = $s['changed']; - if (isset($s['message'])) + if (!empty($s['message'])) $ret['message'] .= $s['message']; } @@ -977,17 +977,15 @@ class Libzot { } } - if (($changed) || ($ud_flags == UPDATE_FLAGS_FORCED)) { + if ($changed/* || ($ud_flags == UPDATE_FLAGS_FORCED)*/) { $guid = random_string() . '@' . \App::get_hostname(); Libzotdir::update_modtime($xchan_hash, $guid, $address, $ud_flags); - logger('Changed: ' . $what, LOGGER_DEBUG); } elseif (!$ud_flags) { // nothing changed but we still need to update the updates record - q("update updates set ud_flags = ( ud_flags | %d ) where ud_addr = '%s' and (ud_flags & %d) = 0 ", - intval(UPDATE_FLAGS_UPDATED), - dbesc($address), - intval(UPDATE_FLAGS_UPDATED) + q("update updates set ud_flags = 0, ud_date = '%s' where ud_hash = '%s'", + dbesc(datetime_convert()), + dbesc($xchan_hash) ); } diff --git a/Zotlabs/Lib/Libzotdir.php b/Zotlabs/Lib/Libzotdir.php index dfedd0bc8..ed2095059 100644 --- a/Zotlabs/Lib/Libzotdir.php +++ b/Zotlabs/Lib/Libzotdir.php @@ -265,16 +265,36 @@ class Libzotdir { if (is_array($j['transactions']) && count($j['transactions'])) { foreach ($j['transactions'] as $t) { - if (empty($t['hash']) || empty($t['transaction_id']) || empty($t['address'])) { + if (empty($t['hash'])) { continue; } - $r = q("select * from updates where ud_guid = '%s' limit 1", - dbesc($t['transaction_id']) + $r = q("select * from updates where ud_hash = '%s' limit 1", + dbesc($t['hash']) ); - if($r) - continue; + if ($r) { + + if ($r[0]['ud_date'] >= $t['timestamp']) { + continue; + } + + q("UPDATE updates SET ud_flags = 1 WHERE ud_id = %d", + dbesc($r[0]['ud_id']) + ); + } + else { + $t['transaction_id'] = strpos($t['transaction_id'], '@') === false ? $t['transaction_id'] : substr($t['transaction_id'], strpos($t['transaction_id'], '@') + 1); + q("insert into updates ( ud_hash, ud_guid, ud_date, ud_addr, ud_flags ) + values ( '%s', '%s', '%s', '%s', 1 ) ", + dbesc($t['hash']), + dbesc($t['transaction_id']), + dbesc($t['timestamp']), + dbesc($t['address']) + ); + } + +/* $ud_flags = 0; if (is_array($t['flags']) && in_array('deleted',$t['flags'])) $ud_flags |= UPDATE_FLAGS_DELETED; @@ -289,6 +309,7 @@ class Libzotdir { intval($ud_flags), dbesc($t['address']) ); +*/ } } } @@ -313,7 +334,7 @@ class Libzotdir { logger('update_directory_entry: ' . print_r($ud,true), LOGGER_DATA); - if ($ud['ud_addr'] && (! ($ud['ud_flags'] & UPDATE_FLAGS_DELETED))) { + if ($ud['ud_hash'] /* && (! ($ud['ud_flags'] & UPDATE_FLAGS_DELETED))*/) { $success = false; $zf = []; @@ -331,10 +352,11 @@ class Libzotdir { dbesc($zf['data']['primary_location']['url']) ); } - - q("update updates set ud_last = '%s' where ud_addr = '%s'", + } + else { + q("UPDATE updates SET ud_last = '%s' WHERE ud_hash = '%s'", dbesc(datetime_convert()), - dbesc($ud['ud_addr']) + dbesc($ud['ud_hash']) ); } } @@ -430,7 +452,7 @@ class Libzotdir { } $ud_hash = random_string() . '@' . \App::get_hostname(); - self::update_modtime($hash, $ud_hash, channel_reddress($p[0]),(($force) ? UPDATE_FLAGS_FORCED : UPDATE_FLAGS_UPDATED)); + self::update_modtime($hash, $ud_hash, channel_reddress($p[0]), 0); } @@ -657,6 +679,30 @@ class Libzotdir { return; } + $u = q("SELECT ud_id FROM updates WHERE ud_hash = '%s' LIMIT 1", + dbesc($hash) + ); + + if ($u) { + $x = q("UPDATE updates SET ud_date = '%s', ud_guid = '%s', ud_addr = '%s', ud_flags = 0 WHERE ud_id = %d", + dbesc(datetime_convert()), + dbesc(\App::get_hostname()), + dbesc($addr), + intval($u[0]['ud_id']) + ); + + return; + } + + q("INSERT INTO updates (ud_hash, ud_guid, ud_date, ud_addr ) VALUES ( '%s', '%s', '%s', '%s' )", + dbesc($hash), + dbesc(\App::get_hostname()), + dbesc(datetime_convert()), + dbesc($addr) + ); + + return; +/* if($flags) { q("insert into updates (ud_hash, ud_guid, ud_date, ud_flags, ud_addr ) values ( '%s', '%s', '%s', %d, '%s' )", dbesc($hash), @@ -673,6 +719,7 @@ class Libzotdir { intval(UPDATE_FLAGS_UPDATED) ); } +*/ } } diff --git a/Zotlabs/Module/Dirsearch.php b/Zotlabs/Module/Dirsearch.php index c0df8a1e3..633ae69f0 100644 --- a/Zotlabs/Module/Dirsearch.php +++ b/Zotlabs/Module/Dirsearch.php @@ -215,9 +215,11 @@ class Dirsearch extends Controller { if($sync) { $spkt = array('transactions' => array()); - $r = q("select * from updates where ud_date >= '%s' and ud_guid != '' and ud_addr != '' order by ud_date desc", + + $r = q("select * from updates where ud_date >= '%s' order by ud_date desc", dbesc($sync) ); + if($r) { foreach($r as $rr) { $flags = array(); @@ -235,6 +237,7 @@ class Dirsearch extends Controller { ); } } + json_return_and_die($spkt); } else { -- cgit v1.2.3