From 1c249a5b06279ea4a19433d45a8dbe6fd71bcd25 Mon Sep 17 00:00:00 2001 From: friendica Date: Sun, 16 Nov 2014 15:51:42 -0800 Subject: some poller optimisations and a fix for undefined (empty) dbtype which shows up as a mysql error that 'rand' isn't found. --- boot.php | 3 ++- include/dba/dba_driver.php | 8 +++++-- include/externals.php | 2 ++ include/poller.php | 55 ++++++++++++++++++++++++++-------------------- version.inc | 2 +- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/boot.php b/boot.php index 7758d1933..569162221 100755 --- a/boot.php +++ b/boot.php @@ -567,7 +567,8 @@ define ( 'ITEM_VERIFIED', 0x2000); // Signature verification was success define ( 'ITEM_RETAINED', 0x4000); // We looked at this item once to decide whether or not to expire it, and decided not to. define ( 'ITEM_RSS', 0x8000); // Item comes from a feed. Use this to decide whether to link the title // Don't make us evaluate this same item again. -define ( 'DBTYPE_MYSQL', 0 ); + +define ( 'DBTYPE_MYSQL', 0 ); define ( 'DBTYPE_POSTGRES', 1 ); /** diff --git a/include/dba/dba_driver.php b/include/dba/dba_driver.php index 103dc8fcc..1bb45b06d 100755 --- a/include/dba/dba_driver.php +++ b/include/dba/dba_driver.php @@ -24,7 +24,10 @@ function dba_factory($server, $port,$user,$pass,$db,$dbtype,$install = false) { $dba = null; - if($dbtype == 1) { + + $dbtype = intval($dbtype); + + if($dbtype == DBTYPE_POSTGRES) { require_once('include/dba/dba_postgres.php'); if(is_null($port)) $port = 5432; $dba = new dba_postgres($server, $port, $user, $pass, $db, $install); @@ -39,6 +42,7 @@ function dba_factory($server, $port,$user,$pass,$db,$dbtype,$install = false) { $dba = new dba_mysql($server, $port,$user,$pass,$db,$install); } } + define('NULL_DATE', $dba->get_null_date()); define('ACTIVE_DBTYPE', $dbtype); return $dba; @@ -371,7 +375,7 @@ function db_getfunc($f) { if(isset($lookup[$f]) && isset($lookup[$f][ACTIVE_DBTYPE])) return $lookup[$f][ACTIVE_DBTYPE]; - logger('Unable to abstract DB function "'. $f . '"', LOG_DEBUG); + logger('Unable to abstract DB function "'. $f . '" for dbtype ' . ACTIVE_DBTYPE, LOGGER_DEBUG); return $f; } diff --git a/include/externals.php b/include/externals.php index 0be5d0fde..b0f853dc6 100644 --- a/include/externals.php +++ b/include/externals.php @@ -14,6 +14,8 @@ function externals_run($argv, $argc){ $total = 0; $attempts = 0; + logger('externals: startup', LOGGER_DEBUG); + // pull in some public posts diff --git a/include/poller.php b/include/poller.php index 952431926..e13c6829d 100644 --- a/include/poller.php +++ b/include/poller.php @@ -27,10 +27,11 @@ function poller_run($argv, $argc){ // Check for a lockfile. If it exists, but is over an hour old, it's stale. Ignore it. $lockfile = 'store/[data]/poller'; - if ((file_exists($lockfile)) && (filemtime($lockfile) > (time() - 3600))) { - logger("poller: Already running"); + if((file_exists($lockfile)) && (filemtime($lockfile) > (time() - 3600)) + && (! get_config('system','override_poll_lockfile'))) { + logger("poller: Already running"); return; - } + } // Create a lockfile. Needs two vars, but $x doesn't need to contain anything. file_put_contents($lockfile, $x); @@ -52,7 +53,7 @@ function poller_run($argv, $argc){ // expire any expired items $r = q("select id from item where expires != '%s' and expires < %s - and not ( item_restrict & %d )>0 ", + and ( item_restrict & %d ) = 0 ", dbesc(NULL_DATE), db_utcnow(), intval(ITEM_DELETED) @@ -69,7 +70,8 @@ function poller_run($argv, $argc){ // or dead entries. $r = q("select channel_id from channel where channel_dirdate < %s - INTERVAL %s", - db_utcnow(), db_quoteinterval('30 DAY') + db_utcnow(), + db_quoteinterval('30 DAY') ); if($r) { foreach($r as $rr) { @@ -82,7 +84,7 @@ function poller_run($argv, $argc){ // publish any applicable items that were set to be published in the future // (time travel posts) - $r = q("select id from item where ( item_restrict & %d )>0 and created <= %s ", + $r = q("select id from item where ( item_restrict & %d ) > 0 and created <= %s ", intval(ITEM_DELAYED_PUBLISH), db_utcnow() ); @@ -207,7 +209,8 @@ function poller_run($argv, $argc){ $r = q("select xchan_photo_l, xchan_hash from xchan where xchan_photo_l != '' and xchan_photo_m = '' and xchan_photo_date < %s - INTERVAL %s", - db_utcnow(), db_quoteinterval('1 DAY') + db_utcnow(), + db_quoteinterval('1 DAY') ); if($r) { require_once('include/photo/photo_driver.php'); @@ -253,13 +256,13 @@ function poller_run($argv, $argc){ } - $sql_extra = (($manual_id) ? " AND abook_id = $manual_id " : ""); + $sql_extra = (($manual_id) ? " AND abook_id = " . intval($manual_id) . " " : ""); reload_plugins(); $d = datetime_convert(); - //TODO check to see if there are any cronhooks before wasting a process + // TODO check to see if there are any cronhooks before wasting a process if(! $restart) proc_run('php','include/cronhooks.php'); @@ -271,12 +274,12 @@ function poller_run($argv, $argc){ : '' ); - $randfunc = (ACTIVE_DBTYPE == DBTYPE_POSTGRES) ? 'RANDOM()' : 'RAND()'; + $randfunc = db_getfunc('RAND'); - $contacts = q("SELECT abook_id, abook_flags, abook_updated, abook_connected, abook_closeness, abook_xchan, abook_channel - FROM abook LEFT JOIN account on abook_account = account_id + $contacts = q("SELECT abook_id, abook_flags, abook_updated, abook_connected, abook_closeness, abook_xchan, abook_channel, xchan_network + FROM abook LEFT JOIN xchan on abook_xchan = xchan_hash LEFT JOIN account on abook_account = account_id $sql_extra - AND (( abook_flags & %d )>0 OR ( abook_flags = %d )) + AND (( abook_flags & %d ) > 0 OR ( abook_flags = %d )) AND (( account_flags = %d ) OR ( account_flags = %d )) $abandon_sql ORDER BY $randfunc", intval(ABOOK_FLAG_HIDDEN|ABOOK_FLAG_PENDING|ABOOK_FLAG_UNCONNECTED|ABOOK_FLAG_FEED), intval(0), @@ -289,6 +292,9 @@ function poller_run($argv, $argc){ foreach($contacts as $contact) { + if($contact['abook_flags'] & ABOOK_FLAG_SELF) + continue; + $update = false; $t = $contact['abook_updated']; @@ -310,6 +316,9 @@ function poller_run($argv, $argc){ } + if($contact['xchan_network'] !== 'zot') + continue; + if($c == $t) { if(datetime_convert('UTC','UTC', 'now') > datetime_convert('UTC','UTC', $t . " + 1 day")) $update = true; @@ -330,17 +339,12 @@ function poller_run($argv, $argc){ // He's dead, Jim if(strcmp(datetime_convert('UTC','UTC', 'now'),datetime_convert('UTC','UTC', $c . " + 30 day")) > 0) { - $n = q("select xchan_network from xchan where xchan_hash = '%s' limit 1", - dbesc($contact['abook_xchan']) + $r = q("update abook set abook_flags = (abook_flags | %d) where abook_id = %d", + intval(ABOOK_FLAG_ARCHIVED), + intval($contact['abook_id']) ); - if($n && $n[0]['xchan_network'] == 'zot') { - $r = q("update abook set abook_flags = (abook_flags | %d) where abook_id = %d", - intval(ABOOK_FLAG_ARCHIVED), - intval($contact['abook_id']) - ); - $update = false; - continue; - } + $update = false; + continue; } if($contact['abook_flags'] & ABOOK_FLAG_ARCHIVED) { @@ -364,6 +368,9 @@ function poller_run($argv, $argc){ } + if($contact['abook_flags'] & (ABOOK_FLAG_PENDING|ABOOK_FLAG_ARCHIVED|ABOOK_FLAG_IGNORED)) + continue; + if((! $update) && (! $force)) continue; @@ -375,7 +382,7 @@ function poller_run($argv, $argc){ } if($dirmode == DIRECTORY_MODE_SECONDARY || $dirmode == DIRECTORY_MODE_PRIMARY) { - $r = q("select distinct ud_addr, updates.* from updates where not ( ud_flags & %d )>0 and ud_addr != '' and ( ud_last = '%s' OR ud_last > %s - INTERVAL %s ) group by ud_addr ", + $r = q("select distinct ud_addr, updates.* from updates where ( ud_flags & %d ) = 0 and ud_addr != '' and ( ud_last = '%s' OR ud_last > %s - INTERVAL %s ) group by ud_addr ", intval(UPDATE_FLAGS_UPDATED), dbesc(NULL_DATE), db_utcnow(), db_quoteinterval('7 DAY') diff --git a/version.inc b/version.inc index 0ea73e309..664554361 100644 --- a/version.inc +++ b/version.inc @@ -1 +1 @@ -2014-11-15.860 +2014-11-16.861 -- cgit v1.2.3