diff options
| -rwxr-xr-x | include/Scrape.php | 15 | ||||
| -rwxr-xr-x | include/delivery.php | 41 | ||||
| -rwxr-xr-x | include/email.php | 10 | ||||
| -rw-r--r-- | include/html2plain.php | 180 | ||||
| -rwxr-xr-x | include/notifier.php | 26 | ||||
| -rwxr-xr-x | include/poller.php | 30 | ||||
| -rw-r--r-- | include/quoteconvert.php | 132 |
7 files changed, 411 insertions, 23 deletions
diff --git a/include/Scrape.php b/include/Scrape.php index 52405ae2d..e98978f38 100755 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -445,10 +445,19 @@ function probe_url($url, $mode = PROBE_NORMAL) { $adr = imap_rfc822_parse_adrlist($x->to,''); if(isset($adr)) { foreach($adr as $feadr) { - if((strcasecmp($feadr->mailbox,$name) == 0) - &&(strcasecmp($feadr->host,$phost) == 0) + if((strcasecmp($feadr->mailbox,$name) == 0) + &&(strcasecmp($feadr->host,$phost) == 0) && (strlen($feadr->personal))) { - $vcard['fn'] = notags($feadr->personal); + + $personal = imap_mime_header_decode($feadr->personal); + $vcard['fn'] = ""; + foreach($personal as $perspart) + if ($perspart->charset != "default") + $vcard['fn'] .= iconv($perspart->charset, 'UTF-8//IGNORE', $perspart->text); + else + $vcard['fn'] .= $perspart->text; + + $vcard['fn'] = notags($vcard['fn']); } } } diff --git a/include/delivery.php b/include/delivery.php index 677d89388..c7c2e8a3f 100755 --- a/include/delivery.php +++ b/include/delivery.php @@ -1,6 +1,7 @@ <?php require_once("boot.php"); require_once('include/queue_fn.php'); +require_once('include/html2plain.php'); function delivery_run($argv, $argc){ global $a, $db; @@ -8,7 +9,7 @@ function delivery_run($argv, $argc){ if(is_null($a)){ $a = new App; } - + if(is_null($db)) { @include(".htconfig.php"); require_once("dba.php"); @@ -293,7 +294,7 @@ function delivery_run($argv, $argc){ $sql_extra = sprintf(" AND `dfrn-id` = '%s' ", dbesc($contact['issued-id'])); else $sql_extra = sprintf(" AND `issued-id` = '%s' ", dbesc($contact['dfrn-id'])); - + $x = q("SELECT `contact`.*, `contact`.`uid` AS `importer_uid`, `contact`.`pubkey` AS `cpubkey`, `contact`.`prvkey` AS `cprvkey`, @@ -322,14 +323,14 @@ function delivery_run($argv, $argc){ require_once('library/simplepie/simplepie.inc'); logger('mod-delivery: local delivery'); local_delivery($x[0],$atom); - break; + break; } } $deliver_status = dfrn_deliver($owner,$contact,$atom); logger('notifier: dfrn_delivery returns ' . $deliver_status); - + if($deliver_status == (-1)) { logger('notifier: delivery failed: queuing message'); add_to_queue($contact['id'],NETWORK_DFRN,$atom); @@ -382,7 +383,7 @@ function delivery_run($argv, $argc){ case NETWORK_MAIL : case NETWORK_MAIL2: - + if(get_config('system','dfrn_only')) break; // WARNING: does not currently convert to RFC2047 header encodings, etc. @@ -432,9 +433,19 @@ function delivery_run($argv, $argc){ if($reply_to) $headers .= 'Reply-to: ' . $reply_to . "\n"; - $headers .= 'Message-id: <' . $it['uri'] . '>' . "\n"; + + // for testing purposes: Collect exported mails + $file = tempnam("/tmp/friendica/", "mail-out-"); + file_put_contents($file, json_encode($it)); + + $headers .= 'Message-Id: <' . cleanupmessageid($it['uri']). '>' . "\n"; + + //logger("Mail: uri: ".$it['uri']." parent-uri ".$it['parent-uri'], LOGGER_DEBUG); + //logger("Mail: Data: ".print_r($it, true), LOGGER_DEBUG); + //logger("Mail: Data: ".print_r($it, true), LOGGER_DATA); + if($it['uri'] !== $it['parent-uri']) { - $header .= 'References: <' . $it['parent-uri'] . '>' . "\n"; + $headers .= 'References: <' . cleanupmessageid($it['parent-uri']) . '>' . "\n"; if(! strlen($it['title'])) { $r = q("SELECT `title` FROM `item` WHERE `parent-uri` = '%s' LIMIT 1", dbesc($it['parent-uri']) @@ -451,10 +462,12 @@ function delivery_run($argv, $argc){ } } $headers .= 'MIME-Version: 1.0' . "\n"; - $headers .= 'Content-Type: text/html; charset=UTF-8' . "\n"; + //$headers .= 'Content-Type: text/html; charset=UTF-8' . "\n"; + $headers .= 'Content-Type: text/plain; charset=UTF-8' . "\n"; $headers .= 'Content-Transfer-Encoding: 8bit' . "\n\n"; $html = prepare_body($it); - $message = '<html><body>' . $html . '</body></html>'; + //$message = '<html><body>' . $html . '</body></html>'; + $message = html2plain($html); logger('notifier: email delivery to ' . $addr); mail($addr, $subject, $message, $headers); } @@ -473,7 +486,7 @@ function delivery_run($argv, $argc){ if((! $contact['pubkey']) && (! $public_message)) break; - + if($target_item['verb'] === ACTIVITY_DISLIKE) { // unsupported break; @@ -514,6 +527,14 @@ function delivery_run($argv, $argc){ return; } +function cleanupmessageid($messageid) { + global $a; + + if (!strpos($messageid, '@')) + $messageid = str_replace(":", ".", $messageid).'@'.$a->get_hostname(); + return($messageid); +} + if (array_search(__file__,get_included_files())===0){ delivery_run($argv,$argc); killme(); diff --git a/include/email.php b/include/email.php index 3e6a8186d..452682260 100755 --- a/include/email.php +++ b/include/email.php @@ -79,6 +79,10 @@ function email_get_msg($mbox,$uid) { if(! $struc) return $ret; + // for testing purposes: Collect imported mails + // $file = tempnam("/tmp/friendica2/", "mail-in-"); + // file_put_contents($file, json_encode($struc)); + if(! $struc->parts) { $ret['body'] = email_get_part($mbox,$uid,$struc,0); } @@ -99,13 +103,17 @@ function email_get_part($mbox,$uid,$p,$partno) { // $partno = '1', '2', '2.1', '2.1.3', etc for multipart, 0 if simple global $htmlmsg,$plainmsg,$charset,$attachments; - echo $partno; + //echo $partno; // DECODE DATA $data = ($partno) ? @imap_fetchbody($mbox,$uid,$partno, FT_UID|FT_PEEK) : @imap_body($mbox,$uid,FT_UID|FT_PEEK); + // for testing purposes: Collect imported mails + // $file = tempnam("/tmp/friendica2/", "mail-body-"); + // file_put_contents($file, $data); + // Any part may be encoded, even plain text messages, so check everything. if ($p->encoding==4) $data = quoted_printable_decode($data); diff --git a/include/html2plain.php b/include/html2plain.php new file mode 100644 index 000000000..2a4cf6639 --- /dev/null +++ b/include/html2plain.php @@ -0,0 +1,180 @@ +<?php +require_once "html2bbcode.php"; + +function breaklines($line, $level) +{ + $wraplen = 75-$level; + + $newlines = array(); + + do { + $oldline = $line; + + $subline = substr($line, 0, $wraplen); + + $pos = strrpos($subline, ' '); + + if ($pos == 0) + $pos = strpos($line, ' '); + + if (($pos > 0) and strlen($line) > $wraplen) { + $newline = trim(substr($line, 0, $pos)); + if ($level > 0) + $newline = str_repeat(">", $level).' '.$newline; + + $newlines[] = $newline." "; + $line = substr($line, $pos+1); + } + + } while ((strlen($line) > $wraplen) and !($oldline == $line)); + + if ($level > 0) + $line = str_repeat(">", $level).' '.$line; + + $newlines[] = $line; + + + return(implode($newlines, "\n")); +} + +function quotelevel($message) +{ + $lines = explode("\n", $message); + + $newlines = array(); + $level = 0; + foreach($lines as $line) {; + $line = trim($line); + $startquote = false; + while (strpos("*".$line, '[quote]') > 0) { + $level++; + $pos = strpos($line, '[quote]'); + $line = substr($line, 0, $pos).substr($line, $pos+7); + $startquote = true; + } + + $currlevel = $level; + + while (strpos("*".$line, '[/quote]') > 0) { + $level--; + if ($level < 0) + $level = 0; + + $pos = strpos($line, '[/quote]'); + $line = substr($line, 0, $pos).substr($line, $pos+8); + } + + if (!$startquote or ($line != '')) + $newlines[] = breaklines($line, $currlevel); + } + return(implode($newlines, "\n")); +} + +function html2plain($html) +{ + global $lang; + + $message = str_replace("\r", "", $html); + + $doc = new DOMDocument(); + $doc->preserveWhiteSpace = false; + + $message = mb_convert_encoding($message, 'HTML-ENTITIES', "UTF-8"); + + @$doc->loadHTML($message); + + $xpath = new DomXPath($doc); + $list = $xpath->query("//pre"); + foreach ($list as $node) { + $node->nodeValue = str_replace("\n", "\r", $node->nodeValue); + } + + $message = $doc->saveHTML(); + $message = str_replace(array("\n<", ">\n", "\r", "\n", "\xC3\x82\xC2\xA0"), array("<", ">", "<br>", " ", ""), $message); + $message = preg_replace('= [\s]*=i', " ", $message); + + // nach <a href="...">...</a> suchen, die ... miteinander vergleichen und bei Gleichheit durch ein einzelnes ... ersetzen. + $pattern = '/<a.*?href="(.*?)".*?>(.*?)<\/a>/is'; + preg_match_all($pattern, $message, $result, PREG_SET_ORDER); + + foreach ($result as $treffer) { + if ($treffer[1] == $treffer[2]) { + $search = '<a href="'.$treffer[1].'" target="_blank">'.$treffer[1].'</a>'; + $message = str_replace($search, $treffer[1], $message); + } + } + @$doc->loadHTML($message); + + node2bbcode($doc, 'html', array(), '', ''); + node2bbcode($doc, 'body', array(), '', ''); + + // MyBB-Auszeichnungen + node2bbcode($doc, 'span', array('style'=>'text-decoration: underline;'), '_', '_'); + node2bbcode($doc, 'span', array('style'=>'font-style: italic;'), '/', '/'); + node2bbcode($doc, 'span', array('style'=>'font-weight: bold;'), '*', '*'); + + node2bbcode($doc, 'strong', array(), '*', '*'); + node2bbcode($doc, 'b', array(), '*', '*'); + node2bbcode($doc, 'i', array(), '/', '/'); + node2bbcode($doc, 'u', array(), '_', '_'); + + node2bbcode($doc, 'blockquote', array(), '[quote]', "[/quote]\n"); + + node2bbcode($doc, 'br', array(), "\n", ''); + + node2bbcode($doc, 'span', array(), "", ""); + node2bbcode($doc, 'pre', array(), "", ""); + node2bbcode($doc, 'div', array(), "\r", "\r"); + node2bbcode($doc, 'p', array(), "\n", "\n"); + + //node2bbcode($doc, 'ul', array(), "\n[list]", "[/list]\n"); + //node2bbcode($doc, 'ol', array(), "\n[list=1]", "[/list]\n"); + node2bbcode($doc, 'li', array(), "\n* ", "\n"); + + node2bbcode($doc, 'hr', array(), str_repeat("-", 70), ""); + + node2bbcode($doc, 'tr', array(), "\n", ""); + node2bbcode($doc, 'td', array(), "\t", ""); + + node2bbcode($doc, 'h1', array(), "\n\n*", "*\n"); + node2bbcode($doc, 'h2', array(), "\n\n*", "*\n"); + node2bbcode($doc, 'h3', array(), "\n\n*", "*\n"); + node2bbcode($doc, 'h4', array(), "\n\n*", "*\n"); + node2bbcode($doc, 'h5', array(), "\n\n*", "*\n"); + node2bbcode($doc, 'h6', array(), "\n\n*", "*\n"); + + node2bbcode($doc, 'a', array('href'=>'/(.+)/'), ' $1', '', true); + node2bbcode($doc, 'img', array('alt'=>'/(.+)/'), '$1', ''); + node2bbcode($doc, 'img', array('title'=>'/(.+)/'), '$1', ''); + node2bbcode($doc, 'img', array(), '', ''); + node2bbcode($doc, 'img', array('src'=>'/(.+)/'), '[img]$1', '[/img]'); + + $message = $doc->saveHTML(); + + $message = str_replace("[img]", "", $message); + $message = str_replace("[/img]", "", $message); + + // was ersetze ich da? + // Irgendein stoerrisches UTF-Zeug + $message = str_replace(chr(194).chr(160), ' ', $message); + + $message = str_replace(" ", " ", $message); + + // Aufeinanderfolgende DIVs + $message = preg_replace('=\r *\r=i', "\n", $message); + $message = str_replace("\r", "\n", $message); + + $message = strip_tags($message); + + $message = html_entity_decode($message, ENT_QUOTES, 'UTF-8'); + + do { + $oldmessage = $message; + $message = str_replace("\n\n\n", "\n\n", $message); + } while ($oldmessage != $message); + + $message = quotelevel(trim($message)); + + return(trim($message)); +} +?> diff --git a/include/notifier.php b/include/notifier.php index 37bc6dae5..3ad433afd 100755 --- a/include/notifier.php +++ b/include/notifier.php @@ -2,6 +2,7 @@ require_once("boot.php"); require_once('include/queue_fn.php'); +require_once('include/html2plain.php'); /* * This file was at one time responsible for doing all deliveries, but this caused @@ -633,7 +634,7 @@ function notifier_run($argv, $argc){ ); if($r1 && $r1[0]['reply_to']) $reply_to = $r1[0]['reply_to']; - + $subject = (($it['title']) ? email_header_encode($it['title'],'UTF-8') : t("\x28no subject\x29")) ; // only expose our real email address to true friends @@ -646,10 +647,14 @@ function notifier_run($argv, $argc){ if($reply_to) $headers .= 'Reply-to: ' . $reply_to . "\n"; - $headers .= 'Message-id: <' . $it['uri'] . '>' . "\n"; + // for testing purposes: Collect exported mails + $file = tempnam("/tmp/friendica/", "mail-out2-"); + file_put_contents($file, json_encode($it)); + + $headers .= 'Message-Id: <' . cleanupmessageid($it['uri']) . '>' . "\n"; if($it['uri'] !== $it['parent-uri']) { - $header .= 'References: <' . $it['parent-uri'] . '>' . "\n"; + $headers .= 'References: <' . cleanupmessageid($it['parent-uri']) . '>' . "\n"; if(! strlen($it['title'])) { $r = q("SELECT `title` FROM `item` WHERE `parent-uri` = '%s' LIMIT 1", dbesc($it['parent-uri']) @@ -667,10 +672,12 @@ function notifier_run($argv, $argc){ } $headers .= 'MIME-Version: 1.0' . "\n"; - $headers .= 'Content-Type: text/html; charset=UTF-8' . "\n"; + //$headers .= 'Content-Type: text/html; charset=UTF-8' . "\n"; + $headers .= 'Content-Type: text/plain; charset=UTF-8' . "\n"; $headers .= 'Content-Transfer-Encoding: 8bit' . "\n\n"; $html = prepare_body($it); - $message = '<html><body>' . $html . '</body></html>'; + //$message = '<html><body>' . $html . '</body></html>'; + $message = html2plain($html); logger('notifier: email delivery to ' . $addr); mail($addr, $subject, $message, $headers); } @@ -834,6 +841,15 @@ function notifier_run($argv, $argc){ return; } +function cleanupmessageid($messageid) { + global $a; + + if (!strpos($messageid, '@')) + $messageid = str_replace(":", ".", $messageid).'@'.$a->get_hostname(); + + return($messageid); +} + if (array_search(__file__,get_included_files())===0){ notifier_run($argv,$argc); killme(); diff --git a/include/poller.php b/include/poller.php index 3e7a1e9b4..ad15ec244 100755 --- a/include/poller.php +++ b/include/poller.php @@ -1,6 +1,7 @@ <?php require_once("boot.php"); +require_once("include/quoteconvert.php"); function poller_run($argv, $argc){ @@ -455,7 +456,19 @@ function poller_run($argv, $argc){ } continue; } - $datarray['title'] = notags(trim($meta->subject)); + + // Decoding the header + $subject = imap_mime_header_decode($meta->subject); + $datarray['title'] = ""; + foreach($subject as $subpart) + if ($subpart->charset != "default") + $datarray['title'] .= iconv($subpart->charset, 'UTF-8//IGNORE', $subpart->text); + else + $datarray['title'] .= $subpart->text; + + $datarray['title'] = notags(trim($datarray['title'])); + + //$datarray['title'] = notags(trim($meta->subject)); $datarray['created'] = datetime_convert('UTC','UTC',$meta->date); $r = email_get_msg($mbox,$msg_uid); @@ -463,15 +476,24 @@ function poller_run($argv, $argc){ logger("Mail: can't fetch msg ".$msg_uid); continue; } - $datarray['body'] = escape_tags($r['body']); + $datarray['body'] = escape_tags(convertquote($r['body'], false)); logger("Mail: Importing ".$msg_uid); // some mailing lists have the original author as 'from' - add this sender info to msg body. // todo: adding a gravatar for the original author would be cool - if(! stristr($meta->from,$contact['addr'])) - $datarray['body'] = t('From: ') . escape_tags($meta->from) . "\n\n" . $datarray['body']; + if(! stristr($meta->from,$contact['addr'])) { + $from = imap_mime_header_decode($meta->from); + $fromdecoded = ""; + foreach($from as $frompart) + if ($frompart->charset != "default") + $fromdecoded .= iconv($frompart->charset, 'UTF-8//IGNORE', $frompart->text); + else + $fromdecoded .= $frompart->text; + + $datarray['body'] = "[b]".t('From: ') . escape_tags($fromdecoded) . "[/b]\n\n" . $datarray['body']; + } $datarray['uid'] = $importer_uid; $datarray['contact-id'] = $contact['id']; diff --git a/include/quoteconvert.php b/include/quoteconvert.php new file mode 100644 index 000000000..3aee93234 --- /dev/null +++ b/include/quoteconvert.php @@ -0,0 +1,132 @@ +<?php +function convertquote($body, $reply) +{ + // Convert Quotes + $arrbody = explode("\n", trim($body)); + $arrlevel = array(); + + for ($i = 0; $i < count($arrbody); $i++) { + $quotelevel = 0; + $quoteline = $arrbody[$i]; + + while ((strlen($quoteline)>0) and ((substr($quoteline, 0, 1) == '>') + or (substr($quoteline, 0, 1) == ' '))) { + if (substr($quoteline, 0, 1) == '>') + $quotelevel++; + + $quoteline = ltrim(substr($quoteline, 1)); + } + + //echo $quotelevel.'*'.$quoteline."\r\n"; + + $arrlevel[$i] = $quotelevel; + $arrbody[$i] = $quoteline; + } + + $quotelevel = 0; + $previousquote = 0; + $arrbodyquoted = array(); + + for ($i = 0; $i < count($arrbody); $i++) { + + $previousquote = $quotelevel; + $quotelevel = $arrlevel[$i]; + $currline = $arrbody[$i]; + + while ($previousquote < $quotelevel) { + if ($sender != '') { + $quote = "[quote title=$sender]"; + $sender = ''; + } else + $quote = "[quote]"; + + $arrbody[$i] = $quote.$arrbody[$i]; + $previousquote++; + } + + while ($previousquote > $quotelevel) { + $arrbody[$i] = '[/quote]'.$arrbody[$i]; + $previousquote--; + } + + $arrbodyquoted[] = $arrbody[$i]; + } + while ($quotelevel > 0) { + $arrbodyquoted[] = '[/quote]'; + $quotelevel--; + } + + $body = implode("\n", $arrbodyquoted); + + if (strlen($body) > 0) + $body = $body."\n\n"; + + if ($reply) + $body = removetofu($body); + + return($body); +} + +function removetofu($message) +{ + $message = trim($message); + + do { + $oldmessage = $message; + $message = preg_replace('=\[/quote\][\s](.*?)\[quote\]=i', '$1', $message); + $message = str_replace("[/quote][quote]", "", $message); + } while ($message != $oldmessage); + + $quotes = array(); + + $startquotes = 0; + + $start = 0; + + while(($pos = strpos($message, '[quote', $start)) > 0) { + $quotes[$pos] = -1; + $start = $pos + 7; + $startquotes++; + } + + $endquotes = 0; + $start = 0; + + while(($pos = strpos($message, '[/quote]', $start)) > 0) { + $start = $pos + 7; + $endquotes++; + } + + while ($endquotes < $startquotes) { + $message .= '[/quote]'; + ++$endquotes; + } + + $start = 0; + + while(($pos = strpos($message, '[/quote]', $start)) > 0) { + $quotes[$pos] = 1; + $start = $pos + 7; + } + + if (strtolower(substr($message, -8)) != '[/quote]') + return($message); + + krsort($quotes); + + $quotelevel = 0; + $quotestart = 0; + foreach ($quotes as $index => $quote) { + $quotelevel += $quote; + + if (($quotelevel == 0) and ($quotestart == 0)) + $quotestart = $index; + } + + if ($quotestart != 0) { + $message = trim(substr($message, 0, $quotestart))."\n[collapsed]\n".substr($message, $quotestart+7, -8).'[/collapsed]'; + } + + return($message); +} +?> |