more efficient public feed fetching

4 files changed, 30 insertions, 9 deletions

diff --git a/boot.php b/boot.php
index f274b264e..643e860c8 100755
--- a/boot.php
+++ b/boot.php
@@ -47,7 +47,7 @@ define ( 'RED_PLATFORM',            'Red Matrix' );
 define ( 'RED_VERSION',             trim(file_get_contents('version.inc')) . 'R');
 define ( 'ZOT_REVISION',            1     ); 
 
-define ( 'DB_UPDATE_VERSION',       1105  );
+define ( 'DB_UPDATE_VERSION',       1106  );
 
 define ( 'EOL',                    '<br />' . "\r\n"     );
 define ( 'ATOM_TIME',              'Y-m-d\TH:i:s\Z' );
diff --git a/include/externals.php b/include/externals.php
index 2ae78c11a..95ced24bf 100644
--- a/include/externals.php
+++ b/include/externals.php
@@ -25,7 +25,7 @@ function externals_run($argv, $argc){
 			$url = $arr['url'];
 		} 
 		else {
-			$r = q("select site_url from site where site_url != '%s' and site_flags != %d order by rand() limit 1",
+			$r = q("select site_url, site_pull from site where site_url != '%s' and site_flags != %d order by rand() limit 1",
 				dbesc(z_root()),
 				intval(DIRECTORY_MODE_STANDALONE)
 			);
@@ -36,16 +36,27 @@ function externals_run($argv, $argc){
 		$attempts ++;
 
 		if($url) {
-			$days = get_config('externals','since_days');
-			if($days === false)
-				$days = 15;
+			if($r[0]['site_pull'] !== '0000-00-00 00:00:00')
+				$mindate = urlencode($r[0]['site_pull']);
+			else {
+				$days = get_config('externals','since_days');
+				if($days === false)
+					$days = 15;
+				$mindate = urlencode(datetime_convert('','','now - ' . intval($days) . ' days'));
+			}
 
-			$feedurl = $url . '/zotfeed?f=&mindate=' . urlencode(datetime_convert('','','now - ' . intval($days) . ' days'));
+			$feedurl = $url . '/zotfeed?f=&mindate=' . $mindate;
 
 			logger('externals: pulling public content from ' . $feedurl, LOGGER_DEBUG);
-			$x = z_fetch_url($feedurl);
 
+			$x = z_fetch_url($feedurl);
 			if(($x) && ($x['success'])) {
+
+				q("update site set site_pull = '%s' where site_url = '%s limit 1",
+					dbesc(datetime_convert()),
+					dbesc($url)
+				);
+
 				$j = json_decode($x['body'],true);
 				if($j['success'] && $j['messages']) {
 					$sys = get_sys_channel();
diff --git a/install/database.sql b/install/database.sql
index 329db698e..f9562ae09 100644
--- a/install/database.sql
+++ b/install/database.sql
@@ -866,7 +866,8 @@ CREATE TABLE IF NOT EXISTS `site` (
   `site_access` int(11) NOT NULL DEFAULT '0',
   `site_flags` int(11) NOT NULL DEFAULT '0',
   `site_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
-  `site_sync` datetime NOT NULL,
+  `site_pull` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+  `site_sync` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
   `site_directory` char(255) NOT NULL DEFAULT '',
   `site_register` int(11) NOT NULL DEFAULT '0',
   `site_sellpage` char(255) NOT NULL DEFAULT '',
diff --git a/install/update.php b/install/update.php
index 55e404e4e..d7a87fad4 100644
--- a/install/update.php
+++ b/install/update.php
@@ -1,6 +1,6 @@
 <?php
 
-define( 'UPDATE_VERSION' , 1105 );
+define( 'UPDATE_VERSION' , 1106 );
 
 /**
  *
@@ -1173,3 +1173,12 @@ function update_r1104() {
 		return UPDATE_SUCCESS;
 	return UPDATE_FAILED;
 }
+
+function update_r1105() {
+	$r = q("ALTER TABLE `site` ADD `site_pull` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00' AFTER `site_update` ,
+CHANGE `site_sync` `site_sync` DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00', ADD INDEX ( `site_pull` ) ");
+	if($r)
+		return UPDATE_SUCCESS;
+	return UPDATE_FAILED;
+}
+