From 01f4ce96f1fe835db774acffb65b2553e1b44c0a Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Thu, 10 Jan 2019 13:35:27 +0100 Subject: do not count removed channels --- Zotlabs/Module/New_channel.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Zotlabs/Module/New_channel.php b/Zotlabs/Module/New_channel.php index 73cdf4c8c..98aa480fe 100644 --- a/Zotlabs/Module/New_channel.php +++ b/Zotlabs/Module/New_channel.php @@ -134,7 +134,7 @@ class New_channel extends \Zotlabs\Web\Controller { $default_role = ''; $aid = get_account_id(); if($aid) { - $r = q("select count(channel_id) as total from channel where channel_account_id = %d", + $r = q("select count(channel_id) as total from channel where channel_account_id = %d and channel_removed = 0", intval($aid) ); if($r && (! intval($r[0]['total']))) { -- cgit v1.2.3 From b54ddccd7beece51d78b21b4ba9e98c908fec4c5 Mon Sep 17 00:00:00 2001 From: ZotSocial Admin Date: Wed, 9 Jan 2019 21:23:12 -0500 Subject: FIX: memory exhaustion on exceptionally large message queues & multiple Queue.php invocations duplicate work --- Zotlabs/Daemon/Queue.php | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Zotlabs/Daemon/Queue.php b/Zotlabs/Daemon/Queue.php index 8f529ff13..e041804f0 100644 --- a/Zotlabs/Daemon/Queue.php +++ b/Zotlabs/Daemon/Queue.php @@ -61,10 +61,18 @@ class Queue { // or just prior to this query based on recent and long-term delivery history. If we have good reason to believe // the site is permanently down, there's no reason to attempt delivery at all, or at most not more than once // or twice a day. - - $r = q("SELECT * FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s ", + + $r = q("SELECT *,RAND() as rn FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s order by rn limit 1", db_utcnow() ); + while ($r) { + foreach($r as $rv) { + queue_deliver($rv); + } + $r = q("SELECT *,RAND() as rn FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s order by rn limit 1", + db_utcnow() + ); + } } if(! $r) return; -- cgit v1.2.3 From 6791b05a4032a076651f7c8e4790614f0f405a55 Mon Sep 17 00:00:00 2001 From: "DM42.Net (Matt Dent)" Date: Thu, 10 Jan 2019 15:29:24 -0500 Subject: Fix for PGSQL/MYSQL difference --- Zotlabs/Daemon/Queue.php | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Zotlabs/Daemon/Queue.php b/Zotlabs/Daemon/Queue.php index e041804f0..6b525b8c3 100644 --- a/Zotlabs/Daemon/Queue.php +++ b/Zotlabs/Daemon/Queue.php @@ -12,6 +12,15 @@ class Queue { require_once('include/items.php'); require_once('include/bbcode.php'); + switch (DBTYPE_ACTIVE) { + case DBTYPE_MYSQL: + $sqlrandfunc = "RAND()"; + break; + + case DBTYPE_POSTGRESQL: + $sqlrandfunc = "RANDOM()"; + break; + } if($argc > 1) $queue_id = $argv[1]; @@ -62,14 +71,14 @@ class Queue { // the site is permanently down, there's no reason to attempt delivery at all, or at most not more than once // or twice a day. - $r = q("SELECT *,RAND() as rn FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s order by rn limit 1", + $r = q("SELECT *,$sqlrandfunc as rn FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s order by rn limit 1", db_utcnow() ); while ($r) { foreach($r as $rv) { queue_deliver($rv); } - $r = q("SELECT *,RAND() as rn FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s order by rn limit 1", + $r = q("SELECT *,$sqlrandfunc as rn FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s order by rn limit 1", db_utcnow() ); } -- cgit v1.2.3 From 7b30fc4b82da0a64e152cdeb82564f8d52b7177e Mon Sep 17 00:00:00 2001 From: zotlabs Date: Thu, 10 Jan 2019 15:02:58 -0800 Subject: failure to import mail --- include/import.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/import.php b/include/import.php index 53b21c317..3bd8b4105 100644 --- a/include/import.php +++ b/include/import.php @@ -1035,8 +1035,9 @@ function import_mail($channel, $mails, $sync = false) { if(! $m) continue; - $m['aid'] = $channel['channel_account_id']; - $m['uid'] = $channel['channel_id']; + $m['account_id'] = $channel['channel_account_id']; + $m['channel_id'] = $channel['channel_id']; + $mail_id = mail_store($m); if($sync && $mail_id) { Zotlabs\Daemon\Master::Summon(array('Notifier','single_mail',$mail_id)); -- cgit v1.2.3 From caf2c0a6c43f3162fc8f4d758eb06cb2f56b3865 Mon Sep 17 00:00:00 2001 From: "DM42.Net (Matt Dent)" Date: Fri, 11 Jan 2019 07:35:12 -0500 Subject: Use dba_driver.php::db_getfunc() --- Zotlabs/Daemon/Queue.php | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/Zotlabs/Daemon/Queue.php b/Zotlabs/Daemon/Queue.php index 6b525b8c3..814148404 100644 --- a/Zotlabs/Daemon/Queue.php +++ b/Zotlabs/Daemon/Queue.php @@ -12,16 +12,6 @@ class Queue { require_once('include/items.php'); require_once('include/bbcode.php'); - switch (DBTYPE_ACTIVE) { - case DBTYPE_MYSQL: - $sqlrandfunc = "RAND()"; - break; - - case DBTYPE_POSTGRESQL: - $sqlrandfunc = "RANDOM()"; - break; - } - if($argc > 1) $queue_id = $argv[1]; else @@ -71,6 +61,8 @@ class Queue { // the site is permanently down, there's no reason to attempt delivery at all, or at most not more than once // or twice a day. + $sqlrandfunc = db_getfunc('rand'); + $r = q("SELECT *,$sqlrandfunc as rn FROM outq WHERE outq_delivered = 0 and outq_scheduled < %s order by rn limit 1", db_utcnow() ); -- cgit v1.2.3 From 8aa117e4ae550db2c156cec6431098700929ff5f Mon Sep 17 00:00:00 2001 From: "M. Dent" Date: Sat, 12 Jan 2019 09:22:37 +0100 Subject: FIX: memory exhaustion on zot message pickup with large message queue --- include/zot.php | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/include/zot.php b/include/zot.php index 9934dae07..6d9b6aeec 100644 --- a/include/zot.php +++ b/include/zot.php @@ -4924,12 +4924,13 @@ function zot_reply_pickup($data) { /* * Everything is good if we made it here, so find all messages that are going to this location - * and send them all. + * and send them all - or a reasonable number if there are a lot so we don't overflow memory. */ - $r = q("select * from outq where outq_posturl = '%s'", + $r = q("select * from outq where outq_posturl = '%s' limit 100", dbesc($data['callback']) ); + if($r) { logger('mod_zot: successful pickup message received from ' . $data['callback'] . ' ' . count($r) . ' message(s) picked up', LOGGER_DEBUG); @@ -4955,6 +4956,19 @@ function zot_reply_pickup($data) { } } + // It's possible that we have more than 100 messages waiting to be sent. + + // See if there are any more messages in the queue. + $x = q("select *,min(outq_created) as earliest from outq where outq_posturl = '%s'", + dbesc($data['callback']) + ); + + // If so, kick off a new delivery notification for the next batch + if ($x) { + logger("Send additional pickup request.", LOGGER_DEBUG); + queue_deliver($x[0],true); + } + // this is a bit of a hack because we don't have the hubloc_url here, only the callback url. // worst case is we'll end up using aes256cbc if they've got a different post endpoint @@ -4966,6 +4980,8 @@ function zot_reply_pickup($data) { $encrypted = crypto_encapsulate(json_encode($ret),$sitekey,$algorithm); json_return_and_die($encrypted); + // @FIXME: There is a possibility that the transmission will get interrupted + // and fail - in which case this packet of messages will be lost. /* pickup: end */ } -- cgit v1.2.3 From 121fa834d56ab3496c82d4d6c5fd8ba70743c26a Mon Sep 17 00:00:00 2001 From: "DM42.Net (Matt Dent)" Date: Sun, 13 Jan 2019 14:06:36 -0500 Subject: FIX: aggregated query error in MYSQL --- include/zot.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/zot.php b/include/zot.php index 6d9b6aeec..f2aa58ee7 100644 --- a/include/zot.php +++ b/include/zot.php @@ -4959,7 +4959,7 @@ function zot_reply_pickup($data) { // It's possible that we have more than 100 messages waiting to be sent. // See if there are any more messages in the queue. - $x = q("select *,min(outq_created) as earliest from outq where outq_posturl = '%s'", + $x = q("select * from outq order by outq_scheduled where outq_posturl = '%s' limit 1", dbesc($data['callback']) ); -- cgit v1.2.3 From ab074f52fece902d1057f531c5289562a7d5343f Mon Sep 17 00:00:00 2001 From: "DM42.Net (Matt Dent)" Date: Sun, 13 Jan 2019 14:10:35 -0500 Subject: outq_created not outq_scheduled --- include/zot.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/zot.php b/include/zot.php index f2aa58ee7..7467eccde 100644 --- a/include/zot.php +++ b/include/zot.php @@ -4959,7 +4959,7 @@ function zot_reply_pickup($data) { // It's possible that we have more than 100 messages waiting to be sent. // See if there are any more messages in the queue. - $x = q("select * from outq order by outq_scheduled where outq_posturl = '%s' limit 1", + $x = q("select * from outq order by outq_created where outq_posturl = '%s' limit 1", dbesc($data['callback']) ); -- cgit v1.2.3 From b4c1ca88f32420cc1653e11811bad74d6f0231a5 Mon Sep 17 00:00:00 2001 From: "DM42.Net (Matt Dent)" Date: Sun, 13 Jan 2019 14:41:44 -0500 Subject: move order by and rewrite similar in update_queue_item() --- include/queue_fn.php | 4 ++-- include/zot.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/queue_fn.php b/include/queue_fn.php index f7e2922c6..85f98aaf9 100644 --- a/include/queue_fn.php +++ b/include/queue_fn.php @@ -13,7 +13,7 @@ function update_queue_item($id, $add_priority = 0) { return; - $y = q("select min(outq_created) as earliest from outq where outq_posturl = '%s'", + $y = q("select outq_created as earliest from outq where outq_posturl = '%s' order by earliest limit 1", dbesc($x[0]['outq_posturl']) ); @@ -311,4 +311,4 @@ function queue_deliver($outq, $immediate = false) { return; } -} \ No newline at end of file +} diff --git a/include/zot.php b/include/zot.php index 7467eccde..df54f2b27 100644 --- a/include/zot.php +++ b/include/zot.php @@ -4959,7 +4959,7 @@ function zot_reply_pickup($data) { // It's possible that we have more than 100 messages waiting to be sent. // See if there are any more messages in the queue. - $x = q("select * from outq order by outq_created where outq_posturl = '%s' limit 1", + $x = q("select * from outq where outq_posturl = '%s' order by outq_created limit 1", dbesc($data['callback']) ); -- cgit v1.2.3