Merge pull request #45 from annando/master

New HTML to BBCode converter, changes to BBCode, nginx support, ...

5 files changed, 381 insertions, 182 deletions

diff --git a/include/api.php b/include/api.php
index 28d6c3f4b..013f4b97a 100755
--- a/include/api.php
+++ b/include/api.php
@@ -99,6 +99,10 @@
 	 **************************/
 	function api_call(&$a){
 		GLOBAL $API, $called_api;
+
+		// preset
+		$type="json";
+
 		foreach ($API as $p=>$info){
 			if (strpos($a->query_string, $p)===0){
 				$called_api= explode("/",$p);
@@ -109,14 +113,14 @@
 
 				load_contact_links(local_user());
 
-				logger('API call for ' . $a->user['username'] . ': ' . $a->query_string);		
+				logger('API call for ' . $a->user['username'] . ': ' . $a->query_string);
 				logger('API parameters: ' . print_r($_REQUEST,true));
-				$type="json";		
+				$type="json";
 				if (strpos($a->query_string, ".xml")>0) $type="xml";
 				if (strpos($a->query_string, ".json")>0) $type="json";
 				if (strpos($a->query_string, ".rss")>0) $type="rss";
-				if (strpos($a->query_string, ".atom")>0) $type="atom";				
-				
+				if (strpos($a->query_string, ".atom")>0) $type="atom";
+
 				$r = call_user_func($info['func'], $a, $type);
 				if ($r===false) return;
 
@@ -126,8 +130,8 @@
 						header ("Content-Type: text/xml");
 						return '<?xml version="1.0" encoding="UTF-8"?>'."\n".$r;
 						break;
-					case "json": 
-						//header ("Content-Type: application/json");  
+					case "json":
+						//header ("Content-Type: application/json");
 						foreach($r as $rr)
 						    return json_encode($rr);
 						break;
@@ -139,19 +143,20 @@
 						header ("Content-Type: application/atom+xml");
 						return '<?xml version="1.0" encoding="UTF-8"?>'."\n".$r;
 						break;
-						
+
 				}
 				//echo "<pre>"; var_dump($r); die();
 			}
 		}
+		logger('API call not implemented: '.$a->query_string." - ".print_r($_REQUEST,true));
 		$r = '<status><error>not implemented</error></status>';
 		switch($type){
 			case "xml":
 				header ("Content-Type: text/xml");
 				return '<?xml version="1.0" encoding="UTF-8"?>'."\n".$r;
 				break;
-			case "json": 
-				header ("Content-Type: application/json");  
+			case "json":
+				header ("Content-Type: application/json");
 			    return json_encode(array('error' => 'not implemented'));
 				break;
 			case "rss":
@@ -162,7 +167,6 @@
 				header ("Content-Type: application/atom+xml");
 				return '<?xml version="1.0" encoding="UTF-8"?>'."\n".$r;
 				break;
-				
 		}
 	}
 
diff --git a/include/bb2diaspora.php b/include/bb2diaspora.php
index c26b0c334..70d4bec9a 100755
--- a/include/bb2diaspora.php
+++ b/include/bb2diaspora.php
@@ -8,35 +8,54 @@ require_once('include/html2bbcode.php');
 
 // we don't want to support a bbcode specific markdown interpreter
 // and the markdown library we have is pretty good, but provides HTML output.
-// So we'll use that to convert to HTML, then convert the HTML back to bbcode, 
+// So we'll use that to convert to HTML, then convert the HTML back to bbcode,
 // and then clean up a few Diaspora specific constructs.
 
 function diaspora2bb($s) {
 
+	// for testing purposes: Collect raw markdown articles
+	$file = tempnam("/tmp/friendica/", "markdown");
+	file_put_contents($file, $s);
+
 	$s = html_entity_decode($s,ENT_COMPAT,'UTF-8');
-    $s = str_replace("\r","\n",$s);
+
+	// Too many new lines. So deactivated the following line
+	// $s = str_replace("\r","\n",$s);
+	// Simply remove cr.
+	$s = str_replace("\r","",$s);
+
+	// <br/> is invalid. Replace it with the valid expression
+	$s = str_replace("<br/>","<br />",$s);
 
 	$s = preg_replace('/\@\{(.+?)\; (.+?)\@(.+?)\}/','@[url=https://$3/u/$2]$1[/url]',$s);
 
-    $s = preg_replace('/\#([^\s\#])/','\\#$1',$s);
+	// Escaping the hash tags - doesn't always seem to work
+	// $s = preg_replace('/\#([^\s\#])/','\\#$1',$s);
+	// This seems to work
+	$s = preg_replace('/\#([^\s\#])/','&#35;$1',$s);
 
 	$s = Markdown($s);
 
-    $s = str_replace('&#35;','#',$s);
-    $s = str_replace("\n",'<br />',$s);
+	$s = str_replace('&#35;','#',$s);
+
+	$s = str_replace("\n",'<br />',$s);
 
 	$s = html2bbcode($s);
 //	$s = str_replace('&#42;','*',$s);
 
+	// Convert everything that looks like a link to a link
+	$s = preg_replace("/([^\]\=]|^)(https?\:\/\/)([a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1[url=$2$3]$2$3[/url]',$s);
 
-    $s = preg_replace("/\[url\=?(.*?)\]https?:\/\/www.youtube.com\/watch\?v\=(.*?)\[\/url\]/ism",'[youtube]$2[/youtube]',$s); 
-    $s = preg_replace("/\[url\=https?:\/\/www.youtube.com\/watch\?v\=(.*?)\].*?\[\/url\]/ism",'[youtube]$1[/youtube]',$s); 
-	$s = preg_replace("/\[url\=?(.*?)\]https?:\/\/vimeo.com\/([0-9]+)(.*?)\[\/url\]/ism",'[vimeo]$2[/vimeo]',$s); 
-	$s = preg_replace("/\[url\=https?:\/\/vimeo.com\/([0-9]+)\](.*?)\[\/url\]/ism",'[vimeo]$1[/vimeo]',$s); 
-	$s = preg_replace("/([^\]\=]|^)(https?\:\/\/)(vimeo|youtu|www\.youtube|soundcloud)([a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1[url]$2$3$4[/url]',$s);
+	//$s = preg_replace("/([^\]\=]|^)(https?\:\/\/)(vimeo|youtu|www\.youtube|soundcloud)([a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1[url=$2$3$4]$2$3$4[/url]',$s);
+	$s = preg_replace("/\[url\=?(.*?)\]https?:\/\/www.youtube.com\/watch\?v\=(.*?)\[\/url\]/ism",'[youtube]$2[/youtube]',$s);
+	$s = preg_replace("/\[url\=https?:\/\/www.youtube.com\/watch\?v\=(.*?)\].*?\[\/url\]/ism",'[youtube]$1[/youtube]',$s);
+	$s = preg_replace("/\[url\=?(.*?)\]https?:\/\/vimeo.com\/([0-9]+)(.*?)\[\/url\]/ism",'[vimeo]$2[/vimeo]',$s);
+	$s = preg_replace("/\[url\=https?:\/\/vimeo.com\/([0-9]+)\](.*?)\[\/url\]/ism",'[vimeo]$1[/vimeo]',$s);
 	// remove duplicate adjacent code tags
 	$s = preg_replace("/(\[code\])+(.*?)(\[\/code\])+/ism","[code]$2[/code]", $s);
-	$s = scale_diaspora_images($s);
+
+	// Don't show link to full picture (until it is fixed)
+	$s = scale_diaspora_images($s, false);
 
 	return $s;
 }
diff --git a/include/bbcode.php b/include/bbcode.php
index cae867eb8..d639b0c39 100755
--- a/include/bbcode.php
+++ b/include/bbcode.php
@@ -12,20 +12,20 @@ function stripcode_br_cb($s) {
 function tryoembed($match){
 	$url = ((count($match)==2)?$match[1]:$match[2]);
 //	logger("tryoembed: $url");
-	
+
 	$o = oembed_fetch_url($url);
 
 	//echo "<pre>"; var_dump($match, $url, $o); killme();
 
 	if ($o->type=="error") return $match[0];
-	
+
 	$html = oembed_format_object($o);
 	return $html; //oembed_iframe($html,$o->width,$o->height);
-	
+
 }
 
-// [noparse][i]italic[/i][/noparse] turns into 
-// [noparse][ i ]italic[ /i ][/noparse], 
+// [noparse][i]italic[/i][/noparse] turns into
+// [noparse][ i ]italic[ /i ][/noparse],
 // to hide them from parser.
 
 function bb_spacefy($st) {
@@ -36,7 +36,7 @@ function bb_spacefy($st) {
   return $new_str;
 }
 
-// The previously spacefied [noparse][ i ]italic[ /i ][/noparse], 
+// The previously spacefied [noparse][ i ]italic[ /i ][/noparse],
 // now turns back and the [noparse] tags are trimed
 // returning [i]italic[/i]
 
@@ -59,8 +59,8 @@ function bbcode($Text,$preserve_nl = false) {
 	$Text = preg_replace_callback("/\[pre\](.*?)\[\/pre\]/ism", 'bb_spacefy',$Text);
 
 
-	// Extract a single private image which uses data url's since preg has issues with 
-	// large data sizes. Stash it away while we do bbcode conversion, and then put it back 
+	// Extract a single private image which uses data url's since preg has issues with
+	// large data sizes. Stash it away while we do bbcode conversion, and then put it back
 	// in after we've done all the regex matching. We cannot use any preg functions to do this.
 
 	$saved_image = '';
@@ -71,13 +71,13 @@ function bbcode($Text,$preserve_nl = false) {
 		$start_fragment = substr($Text,0,$img_start);
 		$img_start += strlen('[img]');
 		$saved_image = substr($Text,$img_start,$img_end - $img_start);
-		$end_fragment = substr($Text,$img_end + strlen('[/img]'));		
+		$end_fragment = substr($Text,$img_end + strlen('[/img]'));
 //		logger('saved_image: ' . $saved_image,LOGGER_DEBUG);
 		$Text = $start_fragment . '[$#saved_image#$]' . $end_fragment;
 	}
 
 	// If we find any event code, turn it into an event.
-	// After we're finished processing the bbcode we'll 
+	// After we're finished processing the bbcode we'll
 	// replace all of the event code with a reformatted version.
 
 	$ev = bbtoevent($Text);
@@ -105,7 +105,7 @@ function bbcode($Text,$preserve_nl = false) {
 	// Perform URL Search
 
 	$Text = preg_replace("/([^\]\=]|^)(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/ism", '$1<a href="$2" target="external-link">$2</a>', $Text);
-	
+
 	$Text = preg_replace_callback("/\[bookmark\=([^\]]*)\].*?\[\/bookmark\]/ism",'tryoembed',$Text);
 	$Text = preg_replace("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/ism",'[url=$1]$2[/url]',$Text);
 
@@ -118,7 +118,7 @@ function bbcode($Text,$preserve_nl = false) {
 	// Perform MAIL Search
 	$Text = preg_replace("/\[mail\]([$MAILSearchString]*)\[\/mail\]/", '<a href="mailto:$1">$1</a>', $Text);
 	$Text = preg_replace("/\[mail\=([$MAILSearchString]*)\](.*?)\[\/mail\]/", '<a href="mailto:$1">$2</a>', $Text);
-         
+
 	// Check for bold text
 	$Text = preg_replace("(\[b\](.*?)\[\/b\])ism",'<strong>$1</strong>',$Text);
 
@@ -149,21 +149,20 @@ function bbcode($Text,$preserve_nl = false) {
 	$Text = str_replace("[*]", "<li>", $Text);
 	$Text = preg_replace("/\[li\](.*?)\[\/li\]/ism", '<li>$1</li>' ,$Text);
 
-	$Text = preg_replace("/\[list\](.*?)\[\/list\]/ism", '<ul class="listbullet" style="list-style-type: circle;">$1</ul>' ,$Text);
-	$Text = preg_replace("/\[ul\](.*?)\[\/ul\]/ism", '<ul class="listbullet" style="list-style-type: circle;">$1</ul>' 
-,$Text);
-	$Text = preg_replace("/\[list=\](.*?)\[\/list\]/ism", '<ul class="listnone" style="list-style-type: none;">$1</ul>' ,$Text);
-	$Text = preg_replace("/\[list=1\](.*?)\[\/list\]/ism", '<ul class="listdecimal" style="list-style-type: decimal;">$1</ul>' ,$Text);
-	$Text = preg_replace("/\[ol\](.*?)\[\/ol\]/ism", '<ul class="listdecimal" style="list-style-type: decimal;">$1</ul>' 
-,$Text);
-	$Text = preg_replace("/\[list=((?-i)i)\](.*?)\[\/list\]/ism",'<ul class="listlowerroman" style="list-style-type: 
-lower-roman;">$2</ul>' ,$Text);
-	$Text = preg_replace("/\[list=((?-i)I)\](.*?)\[\/list\]/ism", '<ul class="listupperroman" style="list-style-type: 
-upper-roman;">$2</ul>' ,$Text);
-	$Text = preg_replace("/\[list=((?-i)a)\](.*?)\[\/list\]/ism", '<ul class="listloweralpha" style="list-style-type: 
-lower-alpha;">$2</ul>' ,$Text);
-	$Text = preg_replace("/\[list=((?-i)A)\](.*?)\[\/list\]/ism", '<ul class="listupperalpha" style="list-style-type: 
-upper-alpha;">$2</ul>' ,$Text);
+ 	// handle nested lists
+	$endlessloop = 0;
+	while (strpos($Text, "[/list]") and strpos($Text, "[list") and (++$endlessloop < 20)) {
+		$Text = preg_replace("/\[list\](.*?)\[\/list\]/ism", '<ul class="listbullet" style="list-style-type: circle;">$1</ul>' ,$Text);
+		$Text = preg_replace("/\[list=\](.*?)\[\/list\]/ism", '<ul class="listnone" style="list-style-type: none;">$1</ul>' ,$Text);
+		$Text = preg_replace("/\[list=1\](.*?)\[\/list\]/ism", '<ul class="listdecimal" style="list-style-type: decimal;">$1</ul>' ,$Text);
+		$Text = preg_replace("/\[list=((?-i)i)\](.*?)\[\/list\]/ism",'<ul class="listlowerroman" style="list-style-type: lower-roman;">$2</ul>' ,$Text);
+		$Text = preg_replace("/\[list=((?-i)I)\](.*?)\[\/list\]/ism", '<ul class="listupperroman" style="list-style-type: upper-roman;">$2</ul>' ,$Text);
+		$Text = preg_replace("/\[list=((?-i)a)\](.*?)\[\/list\]/ism", '<ul class="listloweralpha" style="list-style-type: lower-alpha;">$2</ul>' ,$Text);
+		$Text = preg_replace("/\[list=((?-i)A)\](.*?)\[\/list\]/ism", '<ul class="listupperalpha" style="list-style-type: upper-alpha;">$2</ul>' ,$Text);
+	}
+
+	$Text = preg_replace("/\[ul\](.*?)\[\/ul\]/ism", '<ul class="listbullet" style="list-style-type: circle;">$1</ul>' ,$Text);
+	$Text = preg_replace("/\[ol\](.*?)\[\/ol\]/ism", '<ul class="listdecimal" style="list-style-type: decimal;">$1</ul>' ,$Text);
 
 	$Text = preg_replace("/\[th\](.*?)\[\/th\]/sm", '<th>$1</th>' ,$Text);
 	$Text = preg_replace("/\[td\](.*?)\[\/td\]/sm", '<td>$1</td>' ,$Text);
@@ -190,21 +189,24 @@ upper-alpha;">$2</ul>' ,$Text);
 	// Check for [code] text
 	$Text = preg_replace("/\[code\](.*?)\[\/code\]/ism","$CodeLayout", $Text);
 
-
-
-
 	// Declare the format for [quote] layout
-	$QuoteLayout = '<blockquote>$1</blockquote>';                     
+	$QuoteLayout = '<blockquote>$1</blockquote>';
 	// Check for [quote] text
-	$Text = preg_replace("/\[quote\](.*?)\[\/quote\]/ism","$QuoteLayout", $Text);
+	// handle nested quotes
+	$endlessloop = 0;
+	while (strpos($Text, "[/quote]") and strpos($Text, "[quote]") and (++$endlessloop < 20))
+		$Text = preg_replace("/\[quote\](.*?)\[\/quote\]/ism","$QuoteLayout", $Text);
 
 	// Check for [quote=Author] text
 
 	$t_wrote = t('$1 wrote:');
 
-	$Text = preg_replace("/\[quote=[\"\']*(.*?)[\"\']*\](.*?)\[\/quote\]/ism", 
-                             "<blockquote><strong>" . $t_wrote . "</strong> $2</blockquote>", 
-                             $Text);         
+	// handle nested quotes
+	$endlessloop = 0;
+	while (strpos($Text, "[/quote]") and strpos($Text, "[quote=") and (++$endlessloop < 20))
+		$Text = preg_replace("/\[quote=[\"\']*(.*?)[\"\']*\](.*?)\[\/quote\]/ism",
+        	                     "<blockquote><strong>" . $t_wrote . "</strong> $2</blockquote>",
+                	             $Text);
 
 	// [img=widthxheight]image source[/img]
 	$Text = preg_replace("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", '<img src="$3" style="height: $2px; width: $1px;" >', $Text);
diff --git a/include/html2bbcode.php b/include/html2bbcode.php
index 8025c336b..4297ff2f9 100755
--- a/include/html2bbcode.php
+++ b/include/html2bbcode.php
@@ -1,127 +1,286 @@
 <?php
+/*
+html2bbcode.php
+Converter for HTML to BBCode
+Made by: ike@piratenpartei.de
+Originally made for the syncom project: http://wiki.piratenpartei.de/Syncom
+					https://github.com/annando/Syncom
+*/
 
-/**
- * html2bbcode
- */
-
-
-function html2bbcode($s) {
-
-
-	// only keep newlines from source that are within pre tags
-
-	$s = stripnl_exceptinpre($s);
-
-
-	// Tags to Find
-
-	$htmltags = array(
-		'/\<pre\>(.*?)\<\/pre\>/is',
-		'/\<p(.*?)\>/is',
-		'/\<\/p\>/is',
-		'/\<b\>(.*?)\<\/b\>/is',
-		'/\<i\>(.*?)\<\/i\>/is',
-		'/\<u\>(.*?)\<\/u\>/is',
-		'/\<ul\>(.*?)\<\/ul\>/is',
-		'/\<li\>(.*?)\<\/li\>/is',
-		'/\<img(.*?)width: *([0-9]+)(.*?)height: *([0-9]+)(.*?)src=\"(.*?)\" (.*?)\>/is',
-		'/\<img(.*?)height: *([0-9]+)(.*?)width: *([0-9]+)(.*?)src=\"(.*?)\" (.*?)\>/is',
-		'/\<img(.*?)src=\"(.*?)\"(.*?)width: *([0-9]+)(.*?)height: *([0-9]+)(.*?)\>/is',
-		'/\<img(.*?)src=\"(.*?)\"(.*?)height: *([0-9]+)(.*?)width: *([0-9]+)(.*?)\>/is',
-		'/\<img(.*?) src=\"(.*?)\" (.*?)\>/is',
-		'/\<div(.*?)\>(.*?)\<\/div\>/is',
-		'/\<br(.*?)\>/is',
-		'/\<strong\>(.*?)\<\/strong\>/is',
-		'/\<a (.*?)href=\"(.*?)\"(.*?)\>(.*?)\<\/a\>/is',
-		'/\<code\>(.*?)\<\/code\>/is',
-		'/\<span style=\"color:(.*?)\"\>(.*?)\<\/span\>/is',
-		'/\<span style=\"font-size:(.*?)\"\>(.*?)\<\/span\>/is',
-		'/\<blockquote\>(.*?)\<\/blockquote\>/is',
-		'/\<video(.*?) src=\"(.*?)\" (.*?)\>(.*?)\<\/video\>/is',
-		'/\<audio(.*?) src=\"(.*?)\" (.*?)\>(.*?)\<\/audio\>/is',
-		'/\<iframe(.*?) src=\"(.*?)\" (.*?)\>(.*?)\<\/iframe\>/is',
-
-	);
-
-	// Replace with
-
-	$bbtags = array(
-		'[code]$1[/code]',
-		'',
-		"\n",
-		'[b]$1[/b]',
-		'[i]$1[/i]',
-		'[u]$1[/u]',
-		'[list]$1[/list]',
-		'[*]$1',
-		'[img=$2x$4]$6[/img]',
-		'[img=$4x$2]$6[/img]',
-		'[img=$4x$6]$2[/img]',
-		'[img=$6x$4]$2[/img]',
-		'[img]$2[/img]',
-		'$2',
-		"\n",
-		'[b]$1[/b]',
-		'[url=$2]$4[/url]',
-		'[code]$1[/code]',
-		'[color="$1"]$2[/color]',
-		'[size=$1]$2[/size]',
-		'[quote]$1[/quote]',
-		'[video]$1[/video]',
-		'[audio]$1[/audio]',
-		'[iframe]$1[/iframe]',
-	);
-
-	// Replace $htmltags in $text with $bbtags
-	$text = preg_replace ($htmltags, $bbtags, $s);
-
-	call_hooks('html2bbcode', $text);
-
-	// Strip all other HTML tags
-	$text = strip_tags($text);
-	return $text;
+function node2bbcode(&$doc, $oldnode, $attributes, $startbb, $endbb)
+{
+	do {
+		$done = node2bbcodesub(&$doc, $oldnode, $attributes, $startbb, $endbb);
+	} while ($done);
+}
+
+function node2bbcodesub(&$doc, $oldnode, $attributes, $startbb, $endbb)
+{
+	$savestart = str_replace('$', '%', $startbb);
+	$replace = false;
+
+	$xpath = new DomXPath($doc);
+
+	$list = $xpath->query("//".$oldnode);
+	foreach ($list as $oldNode) {
+
+		$attr = array();
+		if ($oldNode->attributes->length)
+			foreach ($oldNode->attributes as $attribute)
+				$attr[$attribute->name] = $attribute->value;
+
+		$replace = true;
+
+		$startbb = $savestart;
+
+		$i = 0;
+
+		foreach ($attributes as $attribute => $value) {
+
+			$startbb = str_replace('%'.++$i, '$1', $startbb);
+
+			if (strpos('*'.$startbb, '$1') > 0) {
+
+				if ($replace and (@$attr[$attribute] != '')) {
+
+					$startbb = preg_replace($value, $startbb, $attr[$attribute], -1, $count);
+
+					// If nothing could be changed
+					if ($count == 0)
+						$replace = false;
+				} else
+					$replace = false;
+			} else {
+				if (@$attr[$attribute] != $value)
+					$replace = false;
+			}
+		}
+
+		if ($replace) {
+			$StartCode = $oldNode->ownerDocument->createTextNode($startbb);
+			$EndCode = $oldNode->ownerDocument->createTextNode($endbb);
+
+			$oldNode->parentNode->insertBefore($StartCode, $oldNode);
 
+			if ($oldNode->hasChildNodes()) {
+				foreach ($oldNode->childNodes as $child) {
+					$newNode = $child->cloneNode(true);
+					$oldNode->parentNode->insertBefore($newNode, $oldNode);
+				}
+			}
+
+			$oldNode->parentNode->insertBefore($EndCode, $oldNode);
+			$oldNode->parentNode->removeChild($oldNode);
+		}
+	}
+	return($replace);
 }
 
-function stripnl_exceptinpre($string)
+function deletenode(&$doc, $node)
 {
-    // First, check for <pre> tag
-    if(strpos($string, '<pre>') === false)
-    {
-        return str_replace("\n","", $string);
-    }
-
-    // If there is a <pre>, we have to split by line
-    // and manually replace the linebreaks
-
-    $strArr=explode("\n", $string);
-
-    $output="";
-    $preFound=false;
-
-    // Loop over each line
-    foreach($strArr as $line)
-    {    // See if the line has a <pre>. If it does, set $preFound to true
-        if(strpos($line, "<pre>") !== false)
-        {
-            $preFound=true;
-        }
-        elseif(strpos($line, "</pre>") !== false)
-        {
-            $preFound=false;
-        }
-       
-        // If we are in a pre tag, add line and also add \n, else add the line without \n
-        if($preFound)
-        {
-            $output .= $line . "\n";
-        }
-        else
-        {
-            $output .= $line ;
-        }
-    }
-
-    return $output;
+	$xpath = new DomXPath($doc);
+	$list = $xpath->query("//".$node);
+	foreach ($list as $child)
+		$child->parentNode->removeChild($child);
 }
 
+function html2bbcode($message)
+{
+
+	//$file = tempnam("/tmp/", "html");
+	//file_put_contents($file, $message);
+
+	$message = str_replace("\r", "", $message);
+
+	$message = str_replace(array(
+					"<li><p>",
+					"</p></li>"),
+				array(
+					"<li>",
+					"</li>"),
+				$message);
+
+	// remove namespaces
+	$message = preg_replace('=<(\w+):(.+?)>=', '<removeme>', $message);
+	$message = preg_replace('=</(\w+):(.+?)>=', '</removeme>', $message);
+
+	$doc = new DOMDocument();
+	$doc->preserveWhiteSpace = false;
+
+	$message = mb_convert_encoding($message, 'HTML-ENTITIES', "UTF-8");
+
+	@$doc->loadHTML($message);
+
+	deletenode($doc, 'style');
+	deletenode($doc, 'head');
+	deletenode($doc, 'title');
+	deletenode($doc, 'meta');
+	deletenode($doc, 'xml');
+	deletenode($doc, 'removeme');
+
+	$xpath = new DomXPath($doc);
+	$list = $xpath->query("//pre");
+	foreach ($list as $node)
+		$node->nodeValue = str_replace("\n", "\r", $node->nodeValue);
+
+	$message = $doc->saveHTML();
+	$message = str_replace(array("\n<", ">\n", "\r", "\n", "\xC3\x82\xC2\xA0"), array("<", ">", "<br>", " ", ""), $message);
+	$message = preg_replace('= [\s]*=i', " ", $message);
+	@$doc->loadHTML($message);
+
+	node2bbcode($doc, 'html', array(), "", "");
+	node2bbcode($doc, 'body', array(), "", "");
+
+	// Outlook-Quote - Variant 1
+	node2bbcode($doc, 'p', array('class'=>'MsoNormal', 'style'=>'margin-left:35.4pt'), '[quote]', '[/quote]');
+
+	// Outlook-Quote - Variant 2
+	node2bbcode($doc, 'div', array('style'=>'border:none;border-left:solid blue 1.5pt;padding:0cm 0cm 0cm 4.0pt'), '[quote]', '[/quote]');
+
+	// MyBB-Stuff
+	node2bbcode($doc, 'span', array('style'=>'text-decoration: underline;'), '[u]', '[/u]');
+	node2bbcode($doc, 'span', array('style'=>'font-style: italic;'), '[i]', '[/i]');
+	node2bbcode($doc, 'span', array('style'=>'font-weight: bold;'), '[b]', '[/b]');
+
+	node2bbcode($doc, 'font', array('face'=>'/([\w ]+)/', 'size'=>'/(\d+)/', 'color'=>'/(.+)/'), '[font=$1][size=$2][color=$3]', '[/color][/size][/font]');
+	node2bbcode($doc, 'font', array('size'=>'/(\d+)/', 'color'=>'/(.+)/'), '[size=$1][color=$2]', '[/color][/size]');
+	node2bbcode($doc, 'font', array('face'=>'/([\w ]+)/', 'size'=>'/(.+)/'), '[font=$1][size=$2]', '[/size][/font]');
+	node2bbcode($doc, 'font', array('face'=>'/([\w ]+)/', 'color'=>'/(.+)/'), '[font=$1][color=$3]', '[/color][/font]');
+	node2bbcode($doc, 'font', array('face'=>'/([\w ]+)/'), '[font=$1]', '[/font]');
+	node2bbcode($doc, 'font', array('size'=>'/(\d+)/'), '[size=$1]', '[/size]');
+	node2bbcode($doc, 'font', array('color'=>'/(.+)/'), '[color=$1]', '[/color]');
+
+	node2bbcode($doc, 'span', array('style'=>'/.*color:\s*(.+?)[,;].*/'), '[color="$1"]', '[/color]');
+	node2bbcode($doc, 'span', array('style'=>'/.*font-size:\s*(\d+)/'), '[size=$1]', '[/size]');
+
+	//node2bbcode($doc, 'span', array('style'=>'/.*font-family:\s*(.+?)[,;].*/'), '[font=$1]', '[/font]');
+	//node2bbcode($doc, 'div', array('style'=>'/.*font-family:\s*(.+?)[,;].*font-size:\s*(\d+?)pt.*/'), '[font=$1][size=$2]', '[/size][/font]');
+	//node2bbcode($doc, 'div', array('style'=>'/.*font-family:\s*(.+?)[,;].*font-size:\s*(\d+?)px.*/'), '[font=$1][size=$2]', '[/size][/font]');
+	//node2bbcode($doc, 'div', array('style'=>'/.*font-family:\s*(.+?)[,;].*/'), '[font=$1]', '[/font]');
+
+	node2bbcode($doc, 'strong', array(), '[b]', '[/b]');
+	node2bbcode($doc, 'em', array(), '[i]', '[/i]');
+	node2bbcode($doc, 'b', array(), '[b]', '[/b]');
+	node2bbcode($doc, 'i', array(), '[i]', '[/i]');
+	node2bbcode($doc, 'u', array(), '[u]', '[/u]');
+
+	node2bbcode($doc, 'big', array(), "[size=large]", "[/size]");
+	node2bbcode($doc, 'small', array(), "[size=small]", "[/size]");
+
+	node2bbcode($doc, 'blockquote', array(), '[quote]', '[/quote]');
+
+	node2bbcode($doc, 'br', array(), "\n", '');
+
+	node2bbcode($doc, 'p', array('class'=>'MsoNormal'), "\n", "");
+	node2bbcode($doc, 'div', array('class'=>'MsoNormal'), "\r", "");
+
+	node2bbcode($doc, 'span', array(), "", "");
+
+	node2bbcode($doc, 'span', array(), "", "");
+	node2bbcode($doc, 'pre', array(), "", "");
+	node2bbcode($doc, 'div', array(), "\r", "\r");
+	node2bbcode($doc, 'p', array(), "\n", "\n");
+
+	node2bbcode($doc, 'ul', array(), "[list]", "[/list]");
+	node2bbcode($doc, 'ol', array(), "[list=1]", "[/list]");
+	node2bbcode($doc, 'li', array(), "[*]", "");
+
+	node2bbcode($doc, 'hr', array(), "[hr]", "");
+
+	//node2bbcode($doc, 'table', array(), "", "");
+	//node2bbcode($doc, 'tr', array(), "\n", "");
+	//node2bbcode($doc, 'td', array(), "\t", "");
+	node2bbcode($doc, 'table', array(), "[table]", "[/table]");
+	node2bbcode($doc, 'th', array(), "[th]", "[/th]");
+	node2bbcode($doc, 'tr', array(), "[tr]", "[/tr]");
+	node2bbcode($doc, 'td', array(), "[td]", "[/td]");
+
+	node2bbcode($doc, 'h1', array(), "\n\n[size=xx-large][b]", "[/b][/size]\n");
+	node2bbcode($doc, 'h2', array(), "\n\n[size=x-large][b]", "[/b][/size]\n");
+	node2bbcode($doc, 'h3', array(), "\n\n[size=large][b]", "[/b][/size]\n");
+	node2bbcode($doc, 'h4', array(), "\n\n[size=medium][b]", "[/b][/size]\n");
+	node2bbcode($doc, 'h5', array(), "\n\n[size=small][b]", "[/b][/size]\n");
+	node2bbcode($doc, 'h6', array(), "\n\n[size=x-small][b]", "[/b][/size]\n");
+
+	node2bbcode($doc, 'a', array('href'=>'/(.+)/'), '[url=$1]', '[/url]');
+
+	node2bbcode($doc, 'img', array('src'=>'/(.+)/', 'width'=>'/(\d+)/', 'height'=>'/(\d+)/'), '[img$2x$3]$1', '[/img]');
+	node2bbcode($doc, 'img', array('src'=>'/(.+)/'), '[img]$1', '[/img]');
+
+
+	node2bbcode($doc, 'video', array('src'=>'/(.+)/'), '[video]$1', '[/video]');
+	node2bbcode($doc, 'audio', array('src'=>'/(.+)/'), '[audio]$1', '[/audio]');
+	node2bbcode($doc, 'iframe', array('src'=>'/(.+)/'), '[iframe]$1', '[/iframe]');
+
+	node2bbcode($doc, 'code', array(), '[code]$1', '[/code]');
+
+	$message = $doc->saveHTML();
+
+	// I'm removing something really disturbing
+	// Don't know exactly what it is
+	$message = str_replace(chr(194).chr(160), ' ', $message);
+
+	$message = str_replace("&nbsp;", " ", $message);
+
+	// removing multiple DIVs
+	$message = preg_replace('=\r *\r=i', "\n", $message);
+	$message = str_replace("\r", "\n", $message);
+
+	call_hooks('html2bbcode', $message);
+
+	$message = strip_tags($message);
+
+	$message = html_entity_decode($message, ENT_QUOTES, 'UTF-8');
+
+	$message = str_replace(array("<"), array("&lt;"), $message);
+
+	// remove quotes if they don't make sense
+	$message = preg_replace('=\[/quote\][\s]*\[quote\]=i', "\n", $message);
+
+	$message = preg_replace('=\[quote\]\s*=i', "[quote]", $message);
+	$message = preg_replace('=\s*\[/quote\]=i', "[/quote]", $message);
+
+	do {
+		$oldmessage = $message;
+		$message = str_replace("\n \n", "\n\n", $message);
+	} while ($oldmessage != $message);
+
+	do {
+		$oldmessage = $message;
+		$message = str_replace("\n\n\n", "\n\n", $message);
+	} while ($oldmessage != $message);
+
+	do {
+		$oldmessage = $message;
+		$message = str_replace(array(
+					"[/size]\n\n",
+					"\n[hr]",
+					"[hr]\n",
+					"\n[list",
+					"[/list]\n",
+					"\n[/list]",
+					"[list]\n",
+					"[list=1]\n",
+					"\n[*]"),
+				array(
+					"[/size]\n",
+					"[hr]",
+					"[hr]",
+					"[list",
+					"[/list]",
+					"[/list]",
+					"[list]",
+					"[list=1]",
+					"[*]"),
+				$message);
+	} while ($message != $oldmessage);
+
+	$message = str_replace(array('[b][b]', '[/b][/b]', '[i][i]', '[/i][/i]'),
+		array('[b]', '[/b]', '[i]', '[/i]'), $message);
+
+	// Handling Yahoo style of mails
+	$message = str_replace('[hr][b]From:[/b]', '[quote][b]From:[/b]', $message);
+
+	return(trim($message));
+}
+?>
diff --git a/include/poller.php b/include/poller.php
index 3e10533b3..a6b85f2d2 100755
--- a/include/poller.php
+++ b/include/poller.php
@@ -194,8 +194,8 @@ function poller_run($argv, $argc){
 							$update = true;
 						break;
 				}
-				if((! $update) && (! $force))
-					continue;
+//				if((! $update) && (! $force))
+//					continue;
 			}
 
 			// Check to see if we are running out of memory - if so spawn a new process and kill this one
@@ -371,10 +371,14 @@ function poller_run($argv, $argc){
 			}
 			elseif($contact['network'] === NETWORK_MAIL || $contact['network'] === NETWORK_MAIL2) {
 
+				logger("Mail: Fetching");
+
 				$mail_disabled = ((function_exists('imap_open') && (! get_config('system','imap_disabled'))) ? 0 : 1);
 				if($mail_disabled)
 					continue;
 
+				logger("Mail: Enabled");
+
 				$mbox = null;
 				$x = q("SELECT `prvkey` FROM `user` WHERE `uid` = %d LIMIT 1",
 					intval($importer_uid)
@@ -388,6 +392,7 @@ function poller_run($argv, $argc){
 					openssl_private_decrypt(hex2bin($mailconf[0]['pass']),$password,$x[0]['prvkey']);
 					$mbox = email_connect($mailbox,$mailconf[0]['user'],$password);
 					unset($password);
+					logger("Mail: Connect");
 					if($mbox) {
 						q("UPDATE `mailacct` SET `last_check` = '%s' WHERE `id` = %d AND `uid` = %d LIMIT 1",
 							dbesc(datetime_convert()),
@@ -397,11 +402,16 @@ function poller_run($argv, $argc){
 					}
 				}
 				if($mbox) {
+					logger("Mail: mbox");
 
 					$msgs = email_poll($mbox,$contact['addr']);
 
 					if(count($msgs)) {
+						logger("Mail: Parsing ".count($msgs)." mails.");
+
 						foreach($msgs as $msg_uid) {
+							logger("Mail: Parsing mail ".$msg_uid);
+
 							$datarray = array();
 							$meta = email_msg_meta($mbox,$msg_uid);
 							$headers = email_msg_headers($mbox,$msg_uid);
@@ -436,23 +446,28 @@ function poller_run($argv, $argc){
 							);
 
 							if(count($r)) {
+								logger("Mail: Seen before ".$msg_uid);
 								if($meta->deleted && ! $r[0]['deleted']) {
 									q("UPDATE `item` SET `deleted` = 1, `changed` = '%s' WHERE `id` = %d LIMIT 1",
 										dbesc(datetime_convert()),
 										intval($r[0]['id'])
 									);
-								}		
+								}
 								continue;
 							}
 							$datarray['title'] = notags(trim($meta->subject));
 							$datarray['created'] = datetime_convert('UTC','UTC',$meta->date);
-	
+
 							$r = email_get_msg($mbox,$msg_uid);
-							if(! $r)
+							if(! $r) {
+								logger("Mail: can't fetch msg ".$msg_uid);
 								continue;
+							}
 							$datarray['body'] = escape_tags($r['body']);
 
-							// some mailing lists have the original author as 'from' - add this sender info to msg body. 
+							logger("Mail: Importing ".$msg_uid);
+
+							// some mailing lists have the original author as 'from' - add this sender info to msg body.
 							// todo: adding a gravatar for the original author would be cool
 
 							if(! stristr($meta->from,$contact['addr']))
@@ -469,7 +484,7 @@ function poller_run($argv, $argc){
 							$datarray['author-name'] = $contact['name'];
 							$datarray['author-link'] = 'mailbox';
 							$datarray['author-avatar'] = $contact['photo'];
-						
+
 							$stored_item = item_store($datarray);
 							q("UPDATE `item` SET `last-child` = 0 WHERE `parent-uri` = '%s' AND `uid` = %d",
 								dbesc($datarray['parent-uri']),