initial commit directory sync rewrite - should work but still requires lots of cleanup

7 files changed, 101 insertions, 29 deletions

diff --git a/Zotlabs/Daemon/Cron.php b/Zotlabs/Daemon/Cron.php
index 640f06102..caae0ce53 100644
--- a/Zotlabs/Daemon/Cron.php
+++ b/Zotlabs/Daemon/Cron.php
@@ -3,6 +3,7 @@
 namespace Zotlabs\Daemon;
 
 use Zotlabs\Lib\Libsync;
+use Zotlabs\Lib\Libzotdir;
 
 class Cron {
 
@@ -35,6 +36,17 @@ class Cron {
 
 		logger('cron: start');
 
+		// If this is a directory server, request a sync with an upstream
+		// directory at least once a day, up to once every poll interval.
+		// Pull remote changes and push local changes.
+		// potential issue: how do we keep from creating an endless update loop?
+
+		$dirmode = get_config('system', 'directory_mode');
+
+		if ($dirmode == DIRECTORY_MODE_SECONDARY || $dirmode == DIRECTORY_MODE_PRIMARY) {
+			Libzotdir::sync_directories($dirmode);
+		}
+
 		// run queue delivery process in the background
 
 		Master::Summon(array('Queue'));
diff --git a/Zotlabs/Daemon/Onedirsync.php b/Zotlabs/Daemon/Onedirsync.php
index ea995be9e..89f4a40ed 100644
--- a/Zotlabs/Daemon/Onedirsync.php
+++ b/Zotlabs/Daemon/Onedirsync.php
@@ -19,13 +19,14 @@ class Onedirsync {
 			return;
 		}
 
-		$r = q("select * from updates where ud_id = %d limit 1",
+		$r = q("select * from updates where ud_id = %d",
 			intval($update_id)
 		);
 
 		if (!$r)
 			return;
 
+/*
 		if (($r[0]['ud_flags'] & UPDATE_FLAGS_UPDATED) || (!$r[0]['ud_addr']))
 			return;
 
@@ -47,22 +48,24 @@ class Onedirsync {
 			);
 			return;
 		}
-
+*/
 		// ignore doing an update if this ud_addr refers to a known dead hubloc
 
-		$h = q("select * from hubloc where hubloc_addr = '%s'",
+		$h = q("select * from hubloc where hubloc_addr = '%s' order by hubloc_id desc",
 			dbesc($r[0]['ud_addr']),
 		);
 
 		$h = Libzot::zot_record_preferred($h);
 
 		if (($h) && (($h['hubloc_status'] & HUBLOC_OFFLINE) || $h['hubloc_deleted'] || $h['hubloc_error'])) {
-			q("update updates set ud_flags = ( ud_flags | %d ) where ud_addr = '%s' and ( ud_flags & %d ) = 0 ",
-				intval(UPDATE_FLAGS_DELETED),
-				dbesc($r[0]['ud_addr']),
-				intval(UPDATE_FLAGS_UPDATED)
+			q("update updates set ud_flags = 9 where ud_hash = '%s' and ud_flags != 9",
+				dbesc($r[0]['ud_hash'])
 			);
-			return;
+
+			// 2023-04-12: Flag the entry deleted but try to update anyway since the info is not always correct
+			// This might change after all directory servers run the new code.
+
+			// return;
 		}
 
 		// we might have to pull this out some day, but for now update_directory_entry()
diff --git a/Zotlabs/Daemon/Poller.php b/Zotlabs/Daemon/Poller.php
index 0fdc3da16..537209530 100644
--- a/Zotlabs/Daemon/Poller.php
+++ b/Zotlabs/Daemon/Poller.php
@@ -183,11 +183,20 @@ class Poller {
 		$dirmode = intval(get_config('system', 'directory_mode'));
 
 		if ($dirmode == DIRECTORY_MODE_SECONDARY || $dirmode == DIRECTORY_MODE_PRIMARY) {
+/*
 			$r = q("SELECT u.ud_addr, u.ud_id, u.ud_last FROM updates AS u INNER JOIN (SELECT ud_addr, max(ud_id) AS ud_id FROM updates WHERE ( ud_flags & %d ) = 0 AND ud_addr != '' AND ( ud_last <= '%s' OR ud_last > %s - INTERVAL %s ) GROUP BY ud_addr) AS s ON s.ud_id = u.ud_id ",
 				intval(UPDATE_FLAGS_UPDATED),
 				dbesc(NULL_DATE),
 				db_utcnow(), db_quoteinterval('7 DAY')
 			);
+*/
+
+			$r = q("SELECT * FROM updates WHERE ud_flags = 1 AND (ud_last = '%s' OR ud_last > %s - INTERVAL %s)",
+				dbesc(NULL_DATE),
+				db_utcnow(),
+				db_quoteinterval('7 DAY')
+			);
+
 			if ($r) {
 				foreach ($r as $rr) {
 
diff --git a/Zotlabs/Lib/Libsync.php b/Zotlabs/Lib/Libsync.php
index d52b501e4..56bb411c4 100644
--- a/Zotlabs/Lib/Libsync.php
+++ b/Zotlabs/Lib/Libsync.php
@@ -786,7 +786,7 @@ class Libsync {
 
 		if (isset($arr['locations']) && $arr['locations']) {
 
-			$xisting = q("select * from hubloc where hubloc_hash = '%s'",
+			$xisting = q("select * from hubloc where hubloc_hash = '%s' and hubloc_deleted = 0",
 				dbesc($sender['hash'])
 			);
 
diff --git a/Zotlabs/Lib/Libzot.php b/Zotlabs/Lib/Libzot.php
index a8334595f..b41ba043a 100644
--- a/Zotlabs/Lib/Libzot.php
+++ b/Zotlabs/Lib/Libzot.php
@@ -920,11 +920,11 @@ class Libzot {
 		$s = Libsync::sync_locations($arr, $arr);
 
 		if ($s) {
-			if (isset($s['change_message']))
+			if (!empty($s['change_message']))
 				$what .= $s['change_message'];
-			if (isset($s['changed']))
+			if (!empty($s['changed']))
 				$changed = $s['changed'];
-			if (isset($s['message']))
+			if (!empty($s['message']))
 				$ret['message'] .= $s['message'];
 		}
 
@@ -977,17 +977,15 @@ class Libzot {
 			}
 		}
 
-		if (($changed) || ($ud_flags == UPDATE_FLAGS_FORCED)) {
+		if ($changed/* || ($ud_flags == UPDATE_FLAGS_FORCED)*/) {
 			$guid = random_string() . '@' . \App::get_hostname();
 			Libzotdir::update_modtime($xchan_hash, $guid, $address, $ud_flags);
-			logger('Changed: ' . $what, LOGGER_DEBUG);
 		}
 		elseif (!$ud_flags) {
 			// nothing changed but we still need to update the updates record
-			q("update updates set ud_flags = ( ud_flags | %d ) where ud_addr = '%s' and (ud_flags & %d) = 0 ",
-				intval(UPDATE_FLAGS_UPDATED),
-				dbesc($address),
-				intval(UPDATE_FLAGS_UPDATED)
+			q("update updates set ud_flags = 0, ud_date = '%s' where ud_hash = '%s'",
+				dbesc(datetime_convert()),
+				dbesc($xchan_hash)
 			);
 		}
 
diff --git a/Zotlabs/Lib/Libzotdir.php b/Zotlabs/Lib/Libzotdir.php
index dfedd0bc8..ed2095059 100644
--- a/Zotlabs/Lib/Libzotdir.php
+++ b/Zotlabs/Lib/Libzotdir.php
@@ -265,16 +265,36 @@ class Libzotdir {
 			if (is_array($j['transactions']) && count($j['transactions'])) {
 				foreach ($j['transactions'] as $t) {
 
-					if (empty($t['hash']) || empty($t['transaction_id']) || empty($t['address'])) {
+					if (empty($t['hash'])) {
 						continue;
 					}
 
-					$r = q("select * from updates where ud_guid = '%s' limit 1",
-						dbesc($t['transaction_id'])
+					$r = q("select * from updates where ud_hash = '%s' limit 1",
+						dbesc($t['hash'])
 					);
-					if($r)
-						continue;
 
+					if ($r) {
+
+						if ($r[0]['ud_date'] >= $t['timestamp']) {
+							continue;
+						}
+
+						q("UPDATE updates SET ud_flags = 1 WHERE ud_id = %d",
+							dbesc($r[0]['ud_id'])
+						);
+					}
+					else {
+						$t['transaction_id'] = strpos($t['transaction_id'], '@') === false ? $t['transaction_id'] : substr($t['transaction_id'], strpos($t['transaction_id'], '@') + 1);
+						q("insert into updates ( ud_hash, ud_guid, ud_date, ud_addr, ud_flags )
+							values ( '%s', '%s', '%s', '%s', 1 ) ",
+							dbesc($t['hash']),
+							dbesc($t['transaction_id']),
+							dbesc($t['timestamp']),
+							dbesc($t['address'])
+						);
+					}
+
+/*
 					$ud_flags = 0;
 					if (is_array($t['flags']) && in_array('deleted',$t['flags']))
 						$ud_flags |= UPDATE_FLAGS_DELETED;
@@ -289,6 +309,7 @@ class Libzotdir {
 						intval($ud_flags),
 						dbesc($t['address'])
 					);
+*/
 				}
 			}
 		}
@@ -313,7 +334,7 @@ class Libzotdir {
 
 		logger('update_directory_entry: ' . print_r($ud,true), LOGGER_DATA);
 
-		if ($ud['ud_addr'] && (! ($ud['ud_flags'] & UPDATE_FLAGS_DELETED))) {
+		if ($ud['ud_hash'] /* && (! ($ud['ud_flags'] & UPDATE_FLAGS_DELETED))*/) {
 			$success = false;
 			$zf = [];
 
@@ -331,10 +352,11 @@ class Libzotdir {
 						dbesc($zf['data']['primary_location']['url'])
 					);
 				}
-
-				q("update updates set ud_last = '%s' where ud_addr = '%s'",
+			}
+			else {
+				q("UPDATE updates SET ud_last = '%s' WHERE ud_hash = '%s'",
 					dbesc(datetime_convert()),
-					dbesc($ud['ud_addr'])
+					dbesc($ud['ud_hash'])
 				);
 			}
 		}
@@ -430,7 +452,7 @@ class Libzotdir {
 		}
 
 		$ud_hash = random_string() . '@' . \App::get_hostname();
-		self::update_modtime($hash, $ud_hash, channel_reddress($p[0]),(($force) ? UPDATE_FLAGS_FORCED : UPDATE_FLAGS_UPDATED));
+		self::update_modtime($hash, $ud_hash, channel_reddress($p[0]), 0);
 	}
 
 
@@ -657,6 +679,30 @@ class Libzotdir {
 			return;
 		}
 
+		$u = q("SELECT ud_id FROM updates WHERE ud_hash = '%s' LIMIT 1",
+			dbesc($hash)
+		);
+
+		if ($u) {
+			$x = q("UPDATE updates SET ud_date = '%s', ud_guid = '%s', ud_addr = '%s', ud_flags = 0 WHERE ud_id = %d",
+				dbesc(datetime_convert()),
+				dbesc(\App::get_hostname()),
+				dbesc($addr),
+				intval($u[0]['ud_id'])
+			);
+
+			return;
+		}
+
+		q("INSERT INTO updates (ud_hash, ud_guid, ud_date, ud_addr ) VALUES ( '%s', '%s', '%s', '%s' )",
+			dbesc($hash),
+			dbesc(\App::get_hostname()),
+			dbesc(datetime_convert()),
+			dbesc($addr)
+		);
+
+		return;
+/*
 		if($flags) {
 			q("insert into updates (ud_hash, ud_guid, ud_date, ud_flags, ud_addr ) values ( '%s', '%s', '%s', %d, '%s' )",
 				dbesc($hash),
@@ -673,6 +719,7 @@ class Libzotdir {
 				intval(UPDATE_FLAGS_UPDATED)
 			);
 		}
+*/
 	}
 
 }
diff --git a/Zotlabs/Module/Dirsearch.php b/Zotlabs/Module/Dirsearch.php
index c0df8a1e3..633ae69f0 100644
--- a/Zotlabs/Module/Dirsearch.php
+++ b/Zotlabs/Module/Dirsearch.php
@@ -215,9 +215,11 @@ class Dirsearch extends Controller {
 
 		if($sync) {
 			$spkt = array('transactions' => array());
-			$r = q("select * from updates where ud_date >= '%s' and ud_guid != '' and ud_addr != '' order by ud_date desc",
+
+			$r = q("select * from updates where ud_date >= '%s' order by ud_date desc",
 				dbesc($sync)
 			);
+
 			if($r) {
 				foreach($r as $rr) {
 					$flags = array();
@@ -235,6 +237,7 @@ class Dirsearch extends Controller {
 					);
 				}
 			}
+
 			json_return_and_die($spkt);
 		}
 		else {