Merge branch 'deadsite'

16 files changed, 99 insertions, 29 deletions

diff --git a/boot.php b/boot.php
index 8e0e3b004..3afb7704b 100755
--- a/boot.php
+++ b/boot.php
@@ -50,7 +50,7 @@ define ( 'PLATFORM_NAME',           'hubzilla' );
 define ( 'RED_VERSION',             trim(file_get_contents('version.inc')) . 'H');
 define ( 'ZOT_REVISION',            1     );
 
-define ( 'DB_UPDATE_VERSION',       1155  );
+define ( 'DB_UPDATE_VERSION',       1156  );
 
 /**
  * @brief Constant with a HTML line break.
@@ -333,6 +333,12 @@ define ( 'MENU_ITEM_ZID',       0x0001);
 define ( 'MENU_ITEM_NEWWIN',    0x0002);
 define ( 'MENU_ITEM_CHATROOM',  0x0004);
 
+
+
+define ( 'SITE_TYPE_ZOT',           0);
+define ( 'SITE_TYPE_NOTZOT',        1);
+define ( 'SITE_TYPE_UNKNOWN',       2);
+
 /**
  * Poll/Survey types
  */
diff --git a/include/checksites.php b/include/checksites.php
index e14837255..e9c08c202 100644
--- a/include/checksites.php
+++ b/include/checksites.php
@@ -25,8 +25,9 @@ function checksites_run($argv, $argc){
 	if($days < 1)
 		$days = 30;
 
-	$r = q("select * from site where site_dead = 0 and site_update < %s - INTERVAL %s $sql_options ",
-		db_utcnow(), db_quoteinterval($days . ' DAY')
+	$r = q("select * from site where site_dead = 0 and site_update < %s - INTERVAL %s and site_type = %d $sql_options ",
+		db_utcnow(), db_quoteinterval($days . ' DAY'),
+		intval(SITE_TYPE_ZOT)
 	);
 
 	if(! $r)
diff --git a/include/deliver.php b/include/deliver.php
index 82a1ac6df..86fc90b3f 100644
--- a/include/deliver.php
+++ b/include/deliver.php
@@ -53,16 +53,33 @@ function deliver_run($argv, $argc) {
 							continue;
 						}
 					}
+					else {
+
+						// zot sites should all have a site record, unless they've been dead for as long as 
+						// your site has existed. Since we don't know for sure what these sites are, 
+						// call them unknown
+
+						q("insert into site (site_url, site_update, site_dead, site_type) values ('%s','%s',0,%d) ",
+							dbesc($base),
+							dbesc(datetime_convert()),
+							intval(($r[0]['outq_driver'] === 'post') ? SITE_TYPE_NOTZOT : SITE_TYPE_UNKNOWN)
+						);
+					}
 				}
 			} 
 
 			// "post" queue driver - used for diaspora and friendica-over-diaspora communications.
 
 			if($r[0]['outq_driver'] === 'post') {
+
+
 				$result = z_post_url($r[0]['outq_posturl'],$r[0]['outq_msg']); 
 				if($result['success'] && $result['return_code'] < 300) {
 					logger('deliver: queue post success to ' . $r[0]['outq_posturl'], LOGGER_DEBUG);
-
+					q("update site set site_update = '%s', site_dead = 0 where site_url = '%s' ",
+						dbesc(datetime_convert()),
+						dbesc($site_url)
+					);
 					q("update dreport set status = '%s', dreport_time = '%s' where dreport_queue = '%s' limit 1",
 						dbesc('accepted for delivery'),
 						dbesc(datetime_convert()),
diff --git a/include/dir_fns.php b/include/dir_fns.php
index 38c92dd94..e466b7404 100644
--- a/include/dir_fns.php
+++ b/include/dir_fns.php
@@ -151,16 +151,18 @@ function sync_directories($dirmode) {
 
 	$realm = get_directory_realm();
 	if ($realm == DIRECTORY_REALM) {
-		$r = q("select * from site where (site_flags & %d) > 0 and site_url != '%s' and ( site_realm = '%s' or site_realm = '') ",
+		$r = q("select * from site where (site_flags & %d) > 0 and site_url != '%s' and site_type = %d and ( site_realm = '%s' or site_realm = '') ",
 			intval(DIRECTORY_MODE_PRIMARY|DIRECTORY_MODE_SECONDARY),
 			dbesc(z_root()),
+			intval(SITE_TYPE_ZOT),
 			dbesc($realm)
 		);
 	} else {
-		$r = q("select * from site where (site_flags & %d) > 0 and site_url != '%s' and site_realm like '%s' ",
+		$r = q("select * from site where (site_flags & %d) > 0 and site_url != '%s' and site_realm like '%s' and site_type = %d ",
 			intval(DIRECTORY_MODE_PRIMARY|DIRECTORY_MODE_SECONDARY),
 			dbesc(z_root()),
-			dbesc(protect_sprintf('%' . $realm . '%'))
+			dbesc(protect_sprintf('%' . $realm . '%')),
+			intval(SITE_TYPE_ZOT)
 		);
 	}
 
@@ -187,9 +189,10 @@ function sync_directories($dirmode) {
 			intval($r[0]['site_valid'])
 		);
 
-		$r = q("select * from site where (site_flags & %d) > 0 and site_url != '%s'",
+		$r = q("select * from site where (site_flags & %d) > 0 and site_url != '%s' and site_type = %d ",
 			intval(DIRECTORY_MODE_PRIMARY|DIRECTORY_MODE_SECONDARY),
-			dbesc(z_root())
+			dbesc(z_root()),
+			intval(SITE_TYPE_ZOT)
 		);
 	}
 	if (! $r)
diff --git a/include/externals.php b/include/externals.php
index 3b6d170d5..18c034bb2 100644
--- a/include/externals.php
+++ b/include/externals.php
@@ -28,9 +28,10 @@ function externals_run($argv, $argc){
 		} 
 		else {
 			$randfunc = db_getfunc('RAND');
-			$r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d order by $randfunc limit 1",
+			$r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d and site_type = %d order by $randfunc limit 1",
 				dbesc(z_root()),
-				intval(DIRECTORY_MODE_STANDALONE)
+				intval(DIRECTORY_MODE_STANDALONE),
+				intval(SITE_TYPE_ZOT)
 			);
 			if($r)
 				$url = $r[0]['site_url'];
diff --git a/include/hubloc.php b/include/hubloc.php
index 674a3885f..a1171b0e2 100644
--- a/include/hubloc.php
+++ b/include/hubloc.php
@@ -16,7 +16,9 @@ function is_matrix_url($url) {
 
 function prune_hub_reinstalls() {
 
-	$r = q("select site_url from site where true");
+	$r = q("select site_url from site where site_type = %d",
+		intval(SITE_TYPE_ZOT)
+	);
 	if($r) {
 		foreach($r as $rr) {
 			$x = q("select count(*) as t, hubloc_sitekey, max(hubloc_connected) as c from hubloc where hubloc_url = '%s' group by hubloc_sitekey order by c",
diff --git a/include/queue.php b/include/queue.php
index 7d2ad3b2d..18c8c5fa2 100644
--- a/include/queue.php
+++ b/include/queue.php
@@ -22,6 +22,22 @@ function queue_run($argv, $argc){
 
 	logger('queue: start');
 
+
+	$r = q("select outq_posturl from outq where outq_created < %s - INTERVAL %s",
+		db_utcnow(), db_quoteinterval('3 DAY')
+	);
+	if($r) {
+		foreach($r as $rr) {
+			$site_url = '';
+			$h = parse_url($rr['outq_posturl']);
+			$desturl = $h['scheme'] . '://' . $h['host'] . (($h['port']) ? ':' . $h['port'] : '');
+			q("update site set site_dead = 1 where site_dead = 0 and site_url = '%s' and site_update < %s - INTERVAL %s",
+				dbesc($desturl),
+				db_utcnow(), db_quoteinterval('1 MONTH')
+			);
+		}
+	}
+
 	$r = q("DELETE FROM outq WHERE outq_created < %s - INTERVAL %s",
 		db_utcnow(), db_quoteinterval('3 DAY')
 	);
diff --git a/include/zot.php b/include/zot.php
index 0c6a2f368..7c9bdca97 100644
--- a/include/zot.php
+++ b/include/zot.php
@@ -2743,7 +2743,7 @@ function import_site($arr, $pubkey) {
 //			logger('import_site: input: ' . print_r($arr,true));
 //			logger('import_site: stored: ' . print_r($siterecord,true));
 
-			$r = q("update site set site_dead = 0, site_location = '%s', site_flags = %d, site_access = %d, site_directory = '%s', site_register = %d, site_update = '%s', site_sellpage = '%s', site_realm = '%s'
+			$r = q("update site set site_dead = 0, site_location = '%s', site_flags = %d, site_access = %d, site_directory = '%s', site_register = %d, site_update = '%s', site_sellpage = '%s', site_realm = '%s', site_type = %d
 				where site_url = '%s'",
 				dbesc($site_location),
 				intval($site_directory),
@@ -2753,6 +2753,7 @@ function import_site($arr, $pubkey) {
 				dbesc(datetime_convert()),
 				dbesc($sellpage),
 				dbesc($site_realm),
+				intval(SITE_TYPE_ZOT),
 				dbesc($url)
 			);
 			if(! $r) {
@@ -2769,7 +2770,7 @@ function import_site($arr, $pubkey) {
 	}
 	else {
 		$update = true;
-		$r = q("insert into site ( site_location, site_url, site_access, site_flags, site_update, site_directory, site_register, site_sellpage, site_realm )
+		$r = q("insert into site ( site_location, site_url, site_access, site_flags, site_update, site_directory, site_register, site_sellpage, site_realm, site_type )
 			values ( '%s', '%s', %d, %d, '%s', '%s', %d, '%s', '%s' )",
 			dbesc($site_location),
 			dbesc($url),
@@ -2779,7 +2780,8 @@ function import_site($arr, $pubkey) {
 			dbesc($directory_url),
 			intval($register_policy),
 			dbesc($sellpage),
-			dbesc($site_realm)
+			dbesc($site_realm),
+			intval(SITE_TYPE_ZOT)
 		);
 		if(! $r) {
 			logger('import_site: record create failed. ' . print_r($arr,true));
diff --git a/install/schema_mysql.sql b/install/schema_mysql.sql
index c78259e41..0e626752d 100644
--- a/install/schema_mysql.sql
+++ b/install/schema_mysql.sql
@@ -1131,6 +1131,7 @@ CREATE TABLE IF NOT EXISTS `site` (
   `site_realm` char(255) NOT NULL DEFAULT '',
   `site_valid` smallint NOT NULL DEFAULT '0',
   `site_dead` smallint NOT NULL DEFAULT '0',
+  `site_type` smallint NOT NULL DEFAULT '0',
   PRIMARY KEY (`site_url`),
   KEY `site_flags` (`site_flags`),
   KEY `site_update` (`site_update`),
@@ -1141,7 +1142,8 @@ CREATE TABLE IF NOT EXISTS `site` (
   KEY `site_pull` (`site_pull`),
   KEY `site_realm` (`site_realm`),
   KEY `site_valid` (`site_valid`),
-  KEY `site_dead` (`site_dead`)
+  KEY `site_dead` (`site_dead`),
+  KEY `site_type` (`site_type`)
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8;
 
 CREATE TABLE IF NOT EXISTS `source` (
diff --git a/install/schema_postgres.sql b/install/schema_postgres.sql
index 6145e9ccb..f378a3e3d 100644
--- a/install/schema_postgres.sql
+++ b/install/schema_postgres.sql
@@ -1122,6 +1122,7 @@ CREATE TABLE "site" (
   "site_realm" text NOT NULL DEFAULT '',
   "site_valid" smallint NOT NULL DEFAULT '0',
   "site_dead" smallint NOT NULL DEFAULT '0',
+  "site_type" smallint NOT NULL DEFAULT '0',
   PRIMARY KEY ("site_url")
 );
 create index "site_flags" on site ("site_flags");
@@ -1133,6 +1134,7 @@ create index "site_sellpage" on site ("site_sellpage");
 create index "site_realm" on site ("site_realm");
 create index "site_valid" on site ("site_valid");
 create index "site_dead" on site ("site_dead");
+create index "site_type" on site ("site_type");
 
 CREATE TABLE "source" (
   "src_id" serial  NOT NULL,
diff --git a/install/update.php b/install/update.php
index 9f4457600..059d69d14 100644
--- a/install/update.php
+++ b/install/update.php
@@ -1,6 +1,6 @@
 <?php
 
-define( 'UPDATE_VERSION' , 1155 );
+define( 'UPDATE_VERSION' , 1156 );
 
 /**
  *
@@ -1875,3 +1875,12 @@ function update_r1154() {
 
 }
 
+
+function update_r1155() {
+
+	$r1 = q("alter table site add site_type smallint not null default '0' ");
+	$r2 = q("create index site_type on site ( site_type ) ");
+	if($r1 && $r2)
+		return UPDATE_SUCCESS;
+	return UPDATE_FAILED;
+}
diff --git a/mod/dirsearch.php b/mod/dirsearch.php
index 132ed252b..ec730b28d 100644
--- a/mod/dirsearch.php
+++ b/mod/dirsearch.php
@@ -398,13 +398,15 @@ function list_public_sites() {
 	$rand = db_getfunc('rand');
 	$realm = get_directory_realm();
 	if($realm == DIRECTORY_REALM) {
-		$r = q("select * from site where site_access != 0 and site_register !=0 and ( site_realm = '%s' or site_realm = '') order by $rand",
-			dbesc($realm)
+		$r = q("select * from site where site_access != 0 and site_register !=0 and ( site_realm = '%s' or site_realm = '') and site_type = %d order by $rand",
+			dbesc($realm),
+			intval(SITE_TYPE_ZOT)
 		);
 	}
 	else {
-		$r = q("select * from site where site_access != 0 and site_register !=0 and site_realm = '%s' order by $rand",
-			dbesc($realm)
+		$r = q("select * from site where site_access != 0 and site_register !=0 and site_realm = '%s' and site_type = %d order by $rand",
+			dbesc($realm),
+			intval(SITE_TYPE_ZOT)
 		);
 	}
 		
diff --git a/mod/rate.php b/mod/rate.php
index a3a36b4a9..f30afed64 100644
--- a/mod/rate.php
+++ b/mod/rate.php
@@ -22,8 +22,9 @@ function rate_init(&$a) {
 			$a->poi = $r[0];
 		}
 		else {
-			$r = q("select * from site where site_url like '%s' ",
-				dbesc('%' . $target)
+			$r = q("select * from site where site_url like '%s' and site_type = %d",
+				dbesc('%' . $target),
+				intval(SITE_TYPE_ZOT)
 			);
 			if($r) {
 				$a->data['site'] = $r[0];
diff --git a/mod/ratingsearch.php b/mod/ratingsearch.php
index 9c4f2f827..173dd39ff 100644
--- a/mod/ratingsearch.php
+++ b/mod/ratingsearch.php
@@ -35,8 +35,9 @@ function ratingsearch_init(&$a) {
 	if($p)
 		$target = $p[0]['xchan_hash'];
 	else {
-		$p = q("select * from site where site_url like '%s' ",
-			dbesc('%' . $hash)
+		$p = q("select * from site where site_url like '%s' and site_type = %d ",
+			dbesc('%' . $hash),
+			intval(SITE_TYPE_ZOT)
 		);
 		if($p) {
 			$target = strtolower($hash);
diff --git a/mod/regdir.php b/mod/regdir.php
index dce50e76a..7168c3373 100644
--- a/mod/regdir.php
+++ b/mod/regdir.php
@@ -86,8 +86,9 @@ function regdir_init(&$a) {
 		if ($dirmode == DIRECTORY_MODE_STANDALONE) {
 			$r = array(array('site_url' => z_root()));
 		} else {
-			$r = q("select site_url from site where site_flags in ( 1, 2 ) and site_realm = '%s' $sql_extra ",
-				dbesc(get_directory_realm())
+			$r = q("select site_url from site where site_flags in ( 1, 2 ) and site_realm = '%s' and site_type = %d $sql_extra ",
+				dbesc(get_directory_realm()),
+				intval(SITE_TYPE_ZOT)
 			);
 		}
 		if ($r) {
diff --git a/mod/sitelist.php b/mod/sitelist.php
index 12911cbd4..30b2359ac 100644
--- a/mod/sitelist.php
+++ b/mod/sitelist.php
@@ -28,7 +28,9 @@ function sitelist_init(&$a) {
 
 	$result = array('success' => false);
 
-	$r = q("select count(site_url) as total from site where true $sql_extra ");
+	$r = q("select count(site_url) as total from site where site_type = %d $sql_extra ",
+		intval(SITE_TYPE_ZOT)
+	);
 	
 	if($r)
 		$result['total'] = intval($r[0]['total']);
@@ -36,7 +38,9 @@ function sitelist_init(&$a) {
 	$result['start'] = $start;
 	$result['limit'] = $limit;	
 
-	$r = q("select * from site where true $sql_extra $sql_order $sql_limit");
+	$r = q("select * from site where site_type = %d $sql_extra $sql_order $sql_limit",
+		intval(SITE_TYPE_ZOT)
+	);
 
 	$result['results'] = 0;
 	$result['entries'] = array();