From 506853adcdc0229c5b1b9c96e319f99c589c967f Mon Sep 17 00:00:00 2001 From: Friendika Date: Fri, 26 Aug 2011 07:29:22 -0700 Subject: break up delivery into per-person processes --- include/notifier.php | 140 ++++++++++++++++++--------------------------------- 1 file changed, 49 insertions(+), 91 deletions(-) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index e92a4f6a8..1a3b321cf 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -1,4 +1,5 @@ $birthday )); - if($cmd === 'mail') { + if($mail) { $public_message = false; // mail is not public $body = fix_private_photos($item['body'],$owner['uid']); @@ -286,7 +290,7 @@ function notifier_run($argv, $argc){ '$parent_id' => xmlify($item['parent-uri']) )); } - elseif($cmd === 'suggest') { + elseif($fsuggest) { $public_message = false; // suggestions are not public $sugg_template = get_markup_template('atom_suggest.tpl'); @@ -383,8 +387,23 @@ function notifier_run($argv, $argc){ if($contact['self']) continue; + // potentially more than one recipient. Start a new process and space them out a bit. + // we will deliver single recipient types of message and email receipients here. + + if((! $mail) && (! $fsuggest) && (! $followup)) { + $interval = intval(get_config('system','delivery_interval')); + if(! $interval) + $interval = 2; + + proc_run('php','include/delivery.php',$cmd,$item_id,$contact['id']); + sleep($interval); + continue; + } + $deliver_status = 0; + logger("main delivery by notifier: followup=$followup mail=$mail fsuggest=$fsuggest"); + switch($contact['network']) { case NETWORK_DFRN: logger('notifier: dfrndelivery: ' . $contact['name']); @@ -589,52 +608,19 @@ function notifier_run($argv, $argc){ } } - if((strlen($hub)) && ($public_message)) { - $hubs = explode(',', $hub); - if(count($hubs)) { - foreach($hubs as $h) { - $h = trim($h); - if(! strlen($h)) - continue; - $params = 'hub.mode=publish&hub.url=' . urlencode($a->get_baseurl() . '/dfrn_poll/' . $owner['nickname'] ); - post_url($h,$params); - logger('pubsub: publish: ' . $h . ' ' . $params . ' returned ' . $a->get_curl_code()); - if(count($hubs) > 1) - sleep(7); // try and avoid multiple hubs responding at precisely the same time - } - } - } if($public_message) { - /** - * - * If you have less than 999 dfrn friends and it's a public message, - * we'll just go ahead and push them out securely with dfrn/rino or Diaspora. - * If you've got more than that, you'll have to rely on PuSH delivery. - * - */ - - $max_allowed = ((get_config('system','maxpubdeliver') === false) ? 999 : intval(get_config('system','maxpubdeliver'))); - - /** - * - * Only get the bare essentials and go back for the full record. - * If you've got a lot of friends and we grab all the details at once it could exhaust memory. - * - */ - $r = q("SELECT `id`, `name` FROM `contact` WHERE `network` in ('%s','%s') AND `uid` = %d AND `blocked` = 0 AND `pending` = 0 - AND `rel` != %d ", + AND `rel` != %d order by rand() ", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), intval($owner['uid']), intval(CONTACT_IS_SHARING) ); - if((count($r)) && (($max_allowed == 0) || (count($r) < $max_allowed))) { - + if(count($r)) { logger('pubdeliver: ' . print_r($r,true)); foreach($r as $rr) { @@ -646,63 +632,35 @@ function notifier_run($argv, $argc){ continue; } - $n = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1", - intval($rr['id']) - ); - - if(count($n)) { - $contact = $n[0]; - logger('pubdeliver: network: ' . $contact['network']); + if((! $mail) && (! $fsuggest) && (! $followup)) { + $interval = intval(get_config('system','delivery_interval')); + if(! $interval) + $interval = 2; - switch($contact['network']) { - case NETWORK_DFRN : - logger('notifier: dfrnpubdelivery: ' . $contact['name']); - $deliver_status = dfrn_deliver($owner,$contact,$atom); - break; - case NETWORK_DIASPORA : - require_once('include/diaspora.php'); + proc_run('php','include/delivery.php',$cmd,$item_id,$rr['id']); + sleep($interval); + continue; + } + } + } - logger('notifier: diaspora pubdelivery: ' . $contact['name']); - if(get_config('system','dfrn_only') || (! get_config('system','diaspora_enabled')) || (! $normal_mode)) { - logger('notifier: diaspora pubdelivery not allowed at this time'); - break; - } - - if(! $contact['pubkey']) { - logger('notifier: diaspora pubdelivery: no pubkey'); - break; - } - - if($target_item['verb'] === ACTIVITY_DISLIKE) { - // unsupported - break; - } - elseif(($target_item['deleted']) && ($target_item['verb'] !== ACTIVITY_LIKE)) { - // diaspora delete, - diaspora_send_retraction($target_item,$owner,$contact); - break; - } - elseif($followup) { - // send comments, likes and retractions of likes to owner to relay - diaspora_send_followup($target_item,$owner,$contact); - break; - } - elseif($target_item['parent'] != $target_item['id']) { - // we are the relay - send comments, likes and unlikes to our conversants - diaspora_send_relay($target_item,$owner,$contact); - break; - } - elseif($top_level) { - diaspora_send_status($target_item,$owner,$contact); - break; - } - default: - break; - } + if(strlen($hub)) { + $hubs = explode(',', $hub); + if(count($hubs)) { + foreach($hubs as $h) { + $h = trim($h); + if(! strlen($h)) + continue; + $params = 'hub.mode=publish&hub.url=' . urlencode($a->get_baseurl() . '/dfrn_poll/' . $owner['nickname'] ); + post_url($h,$params); + logger('pubsub: publish: ' . $h . ' ' . $params . ' returned ' . $a->get_curl_code()); + if(count($hubs) > 1) + sleep(7); // try and avoid multiple hubs responding at precisely the same time } } } + } return; -- cgit v1.2.3 From f29f228463d35f574d6d285be0cf337b7d39c541 Mon Sep 17 00:00:00 2001 From: Friendika Date: Sun, 28 Aug 2011 19:22:27 -0700 Subject: bring Diaspora message signing back to the source author - whether they like it or not. --- include/notifier.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index 1a3b321cf..b87aa95b1 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -109,7 +109,8 @@ function notifier_run($argv, $argc){ $uid = $r[0]['uid']; $updated = $r[0]['edited']; - $items = q("SELECT * FROM `item` WHERE `parent` = %d ORDER BY `id` ASC", + $items = q("SELECT `item`.*, `sign`.`signed_text`,`sign`.`signature`,`sign`.`signer` + FROM `item` LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id` WHERE `parent` = %d ORDER BY `id` ASC", intval($parent_id) ); -- cgit v1.2.3 From 846c4cea7c0e3868a63a187ee9a504a031b2a7e4 Mon Sep 17 00:00:00 2001 From: Friendika Date: Sun, 28 Aug 2011 21:41:42 -0700 Subject: implement delivery queue in case notifier gets killed --- include/notifier.php | 45 ++++++++++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 13 deletions(-) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index b87aa95b1..cde156cbd 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -379,11 +379,27 @@ function notifier_run($argv, $argc){ dbesc($recip_str) ); - // delivery loop require_once('include/salmon.php'); + $interval = intval(get_config('system','delivery_interval')); + if(! $interval) + $interval = 2; + + // delivery loop + if(count($r)) { + + foreach($r as $contact) { + if((! $mail) && (! $fsuggest) && (! $followup) && (! $contact['self'])) { + q("insert into deliverq ( `cmd`,`item`,`contact` ) values ('%s', %d, %d )", + dbesc($cmd), + intval($item_id), + intval($contact['id']) + ); + } + } + foreach($r as $contact) { if($contact['self']) continue; @@ -392,13 +408,8 @@ function notifier_run($argv, $argc){ // we will deliver single recipient types of message and email receipients here. if((! $mail) && (! $fsuggest) && (! $followup)) { - $interval = intval(get_config('system','delivery_interval')); - if(! $interval) - $interval = 2; - proc_run('php','include/delivery.php',$cmd,$item_id,$contact['id']); - sleep($interval); - continue; + @time_sleep_until(microtime(true) + (float) $interval); } $deliver_status = 0; @@ -624,6 +635,18 @@ function notifier_run($argv, $argc){ if(count($r)) { logger('pubdeliver: ' . print_r($r,true)); + // throw everything into the queue in case we get killed + + foreach($r as $rr) { + if((! $mail) && (! $fsuggest) && (! $followup)) { + q("insert into deliverq ( `cmd`,`item`,`contact` ) values ('%s', %d, %d )", + dbesc($cmd), + intval($item_id), + intval($rr['id']) + ); + } + } + foreach($r as $rr) { /* Don't deliver to folks who have already been delivered to */ @@ -634,13 +657,9 @@ function notifier_run($argv, $argc){ } if((! $mail) && (! $fsuggest) && (! $followup)) { - $interval = intval(get_config('system','delivery_interval')); - if(! $interval) - $interval = 2; - + logger('notifier: delivery agent: ' . $rr['name'] . ' ' . $rr['id']); proc_run('php','include/delivery.php',$cmd,$item_id,$rr['id']); - sleep($interval); - continue; + @time_sleep_until(microtime(true) + (float) $interval); } } } -- cgit v1.2.3 From bfc708e07bde21b772f160aa9873d660f7995c2e Mon Sep 17 00:00:00 2001 From: Friendika Date: Sun, 28 Aug 2011 23:05:06 -0700 Subject: logging on notifier --- include/notifier.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index cde156cbd..aa186317f 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -125,8 +125,10 @@ function notifier_run($argv, $argc){ $item['deleted'] = 1; } - if(count($items) == 1 && $items[0]['uri'] === $items[0]['parent-uri']) + if((count($items) == 1) && ($items[0]['uri'] === $items[0]['parent-uri'])) { + logger('notifier: top level post'); $top_level = true; + } } $r = q("SELECT `contact`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`, -- cgit v1.2.3 From 177e0e15d73fedd3c58a9d5f9cb7f509bd1da07b Mon Sep 17 00:00:00 2001 From: Friendika Date: Mon, 29 Aug 2011 00:59:44 -0700 Subject: allow zero delivery interval on dedicated servers --- include/notifier.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index aa186317f..1c71538d7 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -384,9 +384,7 @@ function notifier_run($argv, $argc){ require_once('include/salmon.php'); - $interval = intval(get_config('system','delivery_interval')); - if(! $interval) - $interval = 2; + $interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval'))); // delivery loop @@ -411,7 +409,8 @@ function notifier_run($argv, $argc){ if((! $mail) && (! $fsuggest) && (! $followup)) { proc_run('php','include/delivery.php',$cmd,$item_id,$contact['id']); - @time_sleep_until(microtime(true) + (float) $interval); + if($interval) + @time_sleep_until(microtime(true) + (float) $interval); } $deliver_status = 0; @@ -661,7 +660,8 @@ function notifier_run($argv, $argc){ if((! $mail) && (! $fsuggest) && (! $followup)) { logger('notifier: delivery agent: ' . $rr['name'] . ' ' . $rr['id']); proc_run('php','include/delivery.php',$cmd,$item_id,$rr['id']); - @time_sleep_until(microtime(true) + (float) $interval); + if($interval) + @time_sleep_until(microtime(true) + (float) $interval); } } } -- cgit v1.2.3 From 7e79b140f038732b340fed70a1d88bdf2b950189 Mon Sep 17 00:00:00 2001 From: Friendika Date: Mon, 29 Aug 2011 17:52:31 -0700 Subject: don't deliver on first pass if already queued --- include/notifier.php | 1 + 1 file changed, 1 insertion(+) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index 1c71538d7..8d2aa961c 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -411,6 +411,7 @@ function notifier_run($argv, $argc){ proc_run('php','include/delivery.php',$cmd,$item_id,$contact['id']); if($interval) @time_sleep_until(microtime(true) + (float) $interval); + continue; } $deliver_status = 0; -- cgit v1.2.3 From 9ff1a3444e9029c94541a7c84f0ed5ba764adef3 Mon Sep 17 00:00:00 2001 From: Friendika Date: Sun, 4 Sep 2011 00:48:45 -0700 Subject: add timezone convert module, several other minor or in progress fixes --- include/notifier.php | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index 8d2aa961c..0bb82b7bf 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -129,6 +129,7 @@ function notifier_run($argv, $argc){ logger('notifier: top level post'); $top_level = true; } + } $r = q("SELECT `contact`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`, @@ -144,6 +145,8 @@ function notifier_run($argv, $argc){ $owner = $r[0]; + $walltowall = ((($top_level) && ($owner['id'] != $items[0]['contact-id'])) ? true : false); + $hub = get_config('system','huburl'); // If this is a public conversation, notify the feed hub -- cgit v1.2.3 From 260d652168bae3716f843f445ae696b59edf4db3 Mon Sep 17 00:00:00 2001 From: Friendika Date: Sun, 4 Sep 2011 05:07:25 -0700 Subject: scrape_url - use feed url if no name can be found --- include/notifier.php | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'include/notifier.php') diff --git a/include/notifier.php b/include/notifier.php index 0bb82b7bf..6ac882c19 100644 --- a/include/notifier.php +++ b/include/notifier.php @@ -2,6 +2,20 @@ require_once("boot.php"); +/* + * This file was at one time responsible for doing all deliveries, but this caused + * big problems on shared hosting systems, where the process might get killed by the + * hosting provider and nothing would get delivered. + * It now only delivers one message under certain cases, and invokes a queued + * delivery mechanism (include/deliver.php) to deliver individual contacts at + * controlled intervals. + * This has a much better chance of surviving random processes getting killed + * by the hosting provider. + * A lot of this code is duplicated in include/deliver.php until we have time to go back + * and re-structure the delivery procedure based on the obstacles that have been thrown at + * us by hosting providers. + */ + function notifier_run($argv, $argc){ global $a, $db; @@ -594,7 +608,8 @@ function notifier_run($argv, $argc){ diaspora_send_relay($target_item,$owner,$contact); break; } - elseif($top_level) { + elseif(($top_level) && (! $walltowall)) { + // currently no workable solution for sending walltowall diaspora_send_status($target_item,$owner,$contact); break; } -- cgit v1.2.3