From b6477cd38b532ebe69a480460e7b883964ec6627 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 9 Mar 2017 11:51:21 -0800
Subject: go back to where the problem started

---
 include/items.php | 70 ++-----------------------------------------------------
 1 file changed, 2 insertions(+), 68 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 6d21953a9..8032f53c4 100755
--- a/include/items.php
+++ b/include/items.php
@@ -600,11 +600,6 @@ function get_item_elements($x,$allow_code = false) {
 
 	$arr['sig']          = (($x['signature']) ? htmlspecialchars($x['signature'],  ENT_COMPAT,'UTF-8',false) : '');
 
-	if(array_key_exists('diaspora_signature',$x) && is_array($x['diaspora_signature']))
-		$x['diaspora_signature'] = json_encode($x['diaspora_signature']);
-
-	$arr['diaspora_meta'] = (($x['diaspora_signature']) ? $x['diaspora_signature'] : '');
-
 	$arr['obj']          = activity_sanitise($x['object']);
 	$arr['target']       = activity_sanitise($x['target']);
 
@@ -1035,17 +1030,7 @@ function encode_item($item,$mirror = false) {
 	if($item['iconfig'])
 		$x['meta']        = encode_item_meta($item['iconfig'],$mirror);
 
-	if($item['diaspora_meta']) {
-		$z = json_decode($item['diaspora_meta'],true);
-		if($z) {
-			if(is_array($z) && array_key_exists('iv',$z))
-				$x['diaspora_signature'] = crypto_unencapsulate($z,$key);
-			else
-				$x['diaspora_signature'] = $z;
-			if(! is_array($z))
-				logger('encode_item: diaspora meta is not an array: ' . print_r($z,true));
-		}
-	}
+
 	logger('encode_item: ' . print_r($x,true), LOGGER_DATA);
 
 	return $x;
@@ -1324,7 +1309,6 @@ function encode_mail($item,$extended = false) {
 	$x['message_parent'] = $item['parent_mid'];
 	$x['created']        = $item['created'];
 	$x['expires']        = $item['expires'];
-	$x['diaspora_meta']  = $item['diaspora_meta'];
 	$x['title']          = $item['title'];
 	$x['body']           = $item['body'];
 	$x['from']           = encode_item_xchan($item['from']);
@@ -1517,7 +1501,6 @@ function item_store($arr, $allow_exec = false, $deliver = true) {
 	$arr['title'] = ((array_key_exists('title',$arr) && strlen($arr['title']))  ? trim($arr['title']) : '');
 	$arr['body']  = ((array_key_exists('body',$arr) && strlen($arr['body']))    ? trim($arr['body'])  : '');
 
-	$arr['diaspora_meta'] = ((x($arr,'diaspora_meta')) ? $arr['diaspora_meta']               : '');
 	$arr['allow_cid']     = ((x($arr,'allow_cid'))     ? trim($arr['allow_cid'])             : '');
 	$arr['allow_gid']     = ((x($arr,'allow_gid'))     ? trim($arr['allow_gid'])             : '');
 	$arr['deny_cid']      = ((x($arr,'deny_cid'))      ? trim($arr['deny_cid'])              : '');
@@ -2027,7 +2010,7 @@ function item_store_update($arr,$allow_exec = false, $deliver = true) {
 	$arr['changed']       = $orig[0]['changed'];
 
 	$arr['route']         = ((array_key_exists('route',$arr)) ? trim($arr['route'])          : $orig[0]['route']);
-	$arr['diaspora_meta'] = ((x($arr,'diaspora_meta')) ? $arr['diaspora_meta']               : $orig[0]['diaspora_meta']);
+
 	$arr['location']      = ((x($arr,'location'))      ? notags(trim($arr['location']))      : $orig[0]['location']);
 	$arr['coord']         = ((x($arr,'coord'))         ? notags(trim($arr['coord']))         : $orig[0]['coord']);
 	$arr['verb']          = ((x($arr,'verb'))          ? notags(trim($arr['verb']))          : $orig[0]['verb']);
@@ -2186,55 +2169,6 @@ function item_store_update($arr,$allow_exec = false, $deliver = true) {
 
 
 
-function store_diaspora_comment_sig($datarray, $channel, $parent_item, $post_id, $walltowall = false) {
-
-	// We won't be able to sign Diaspora comments for authenticated visitors
-	// - we don't have their private key
-
-	// since Diaspora doesn't handle edits we can only do this for the original text and not update it.
-
-	require_once('include/markdown.php');
-	$signed_body = bb2diaspora_itembody($datarray,$walltowall);
-
-	if($walltowall) {
-		logger('wall to wall comment',LOGGER_DEBUG);
-		// post will come across with the owner's identity. Throw a preamble onto the post to indicate the true author.
-		$signed_body = "\n\n"
-			. '![' . $datarray['author']['xchan_name'] . '](' . $datarray['author']['xchan_photo_m'] . ')'
-			. '[' . $datarray['author']['xchan_name'] . '](' . $datarray['author']['xchan_url'] . ')' . "\n\n"
-			. $signed_body;
-	}
-
-	logger('storing diaspora comment signature',LOGGER_DEBUG);
-
-	$diaspora_handle = channel_reddress($channel);
-
-	$signed_text = $datarray['mid'] . ';' . $parent_item['mid'] . ';' . $signed_body . ';' . $diaspora_handle;
-
-
-	if( $channel && $channel['channel_prvkey'] )
-		$authorsig = base64_encode(rsa_sign($signed_text, $channel['channel_prvkey'], 'sha256'));
-	else
-		$authorsig = '';
-
-	$x = array('signer' => $diaspora_handle, 'body' => $signed_body, 'signed_text' => $signed_text, 'signature' => $authorsig);
-
-	$y = json_encode($x);
-
-	$r = q("update item set diaspora_meta = '%s' where id = %d",
-		dbesc($y),
-		intval($post_id)
-	);
-
-
-	if(! $r)
-		logger('store_diaspora_comment_sig: DB write failed');
-
-	return;
-}
-
-
-
 function send_status_notifications($post_id,$item) {
 
 	// only send notifications for comments
-- 
cgit v1.2.3


From c93db2cd218d5a1e8ff6e5b243b99e7fcbae36ea Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 9 Mar 2017 14:31:54 -0800
Subject: more cleanup

---
 include/items.php | 35 +----------------------------------
 1 file changed, 1 insertion(+), 34 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 033863c42..58305aa63 100755
--- a/include/items.php
+++ b/include/items.php
@@ -248,8 +248,6 @@ function can_comment_on_post($observer_xchan, $item) {
 	}
 	if(strstr($item['comment_policy'],'network:') && strstr($item['comment_policy'],'red'))
 		return true;
-	if(strstr($item['comment_policy'],'network:') && strstr($item['comment_policy'],'diaspora'))
-		return true;
 	if(strstr($item['comment_policy'],'site:') && strstr($item['comment_policy'],App::get_hostname()))
 		return true;
 
@@ -766,8 +764,6 @@ function import_author_xchan($x) {
 	if((! array_key_exists('network', $x)) || ($x['network'] === 'zot')) {
 		$y = import_author_zot($x);
 	}
-	if(! $y)
-		$y = import_author_diaspora($x);
 
 	if($x['network'] === 'rss') {
 		$y = import_author_rss($x);
@@ -780,36 +776,6 @@ function import_author_xchan($x) {
 	return(($y) ? $y : false);
 }
 
-/**
- * @brief Imports an author from Diaspora.
- *
- * @param array $x an associative array with
- *   * \e string \b address
- * @return boolean|string false on error, otherwise xchan_hash of the new entry
- */
-function import_author_diaspora($x) {
-	if(! $x['address'])
-		return false;
-
-	$r = q("select * from xchan where xchan_addr = '%s' limit 1",
-		dbesc($x['address'])
-	);
-	if($r) {
-		logger('in_cache: ' . $x['address'], LOGGER_DATA);
-		return $r[0]['xchan_hash'];
-	}
-
-	if(discover_by_webbie($x['address'])) {
-		$r = q("select xchan_hash from xchan where xchan_addr = '%s' limit 1",
-			dbesc($x['address'])
-		);
-		if($r)
-			return $r[0]['xchan_hash'];
-	}
-
-	return false;
-}
-
 /**
  * @brief Imports an author from a RSS feed.
  *
@@ -819,6 +785,7 @@ function import_author_diaspora($x) {
  *   * \e string \b guid
  * @return boolean|string
  */
+
 function import_author_rss($x) {
 	if(! $x['url'])
 		return false;
-- 
cgit v1.2.3


From 2f5f1a4d641c5873c308d6ab58ae239f2758e1e4 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Tue, 14 Mar 2017 16:14:05 -0700
Subject: fix group by item query

---
 include/items.php | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 58305aa63..1037f9814 100755
--- a/include/items.php
+++ b/include/items.php
@@ -3674,7 +3674,7 @@ function zot_feed($uid,$observer_hash,$arr) {
 	if(! is_sys_channel($uid))
 		$sql_extra = item_permissions_sql($uid,$observer_hash);
 
-	$limit = " LIMIT 100 ";
+	$limit = " LIMIT 5000 ";
 
 	if($mindate > NULL_DATE) {
 		$sql_extra .= " and ( created > '$mindate' or changed > '$mindate' ) ";
@@ -3686,15 +3686,7 @@ function zot_feed($uid,$observer_hash,$arr) {
 	}
 
 
-	$items = array();
-
-	/** @FIXME re-unite these SQL statements. There is no need for them to be separate. The mySQL is convoluted with misuse of group by. As it stands, there is a slight difference where the postgres version doesn't remove the duplicate parents up to 100. In practice this doesn't matter. It could be made to match behavior by adding "distinct on (parent) " to the front of the selection list, at a not-worth-it performance penalty (page temp results to disk). duplicates are still ignored in the in() clause, you just get less than 100 parents if there are many children. */
-
-	if(ACTIVE_DBTYPE == DBTYPE_POSTGRES) {
-		$groupby = '';
-	} else {
-		$groupby = 'GROUP BY parent';
-	}
+	$items = [];
 
 	$item_normal = item_normal();
 
@@ -3703,7 +3695,7 @@ function zot_feed($uid,$observer_hash,$arr) {
 			WHERE uid != %d
 			$item_normal
 			AND item_wall = 1
-			and item_private = 0 $sql_extra $groupby ORDER BY created ASC $limit",
+			and item_private = 0 $sql_extra ORDER BY created ASC $limit",
 			intval($uid)
 		);
 	}
@@ -3711,19 +3703,25 @@ function zot_feed($uid,$observer_hash,$arr) {
 		$r = q("SELECT parent, created, postopts from item
 			WHERE uid = %d $item_normal
 			AND item_wall = 1
-			$sql_extra $groupby ORDER BY created ASC $limit",
+			$sql_extra ORDER BY created ASC $limit",
 			intval($uid)
 		);
 	}
 
+	$parents = [];
+
 	if($r) {
-		for($x = 0; $x < count($r); $x ++) {
-			if(strpos($r[$x]['postopts'],'nodeliver') !== false) {
-				unset($r[$x]);
-			}
+		foreach($r as $rv) {
+			if(array_key_exists($rv['parent'],$parents))
+				continue;
+			if(strpos($rv['postopts'],'nodeliver') !== false)
+				continue;
+			$parents[$rv['parent']] = $rv;
+			if(count($parents) > 200)
+				break;
 		}
 
-		$parents_str = ids_to_querystr($r,'parent');
+		$parents_str = ids_to_querystr($parents,'parent');
 		$sys_query = ((is_sys_channel($uid)) ? $sql_extra : '');
 		$item_normal = item_normal();
 
-- 
cgit v1.2.3


From 2c73b457ef0943d46804480a0aa016f64c11edbf Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Tue, 14 Mar 2017 17:07:29 -0700
Subject: input filter updates

---
 include/items.php | 87 +++++++++++++++++++++++--------------------------------
 1 file changed, 37 insertions(+), 50 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 1037f9814..c978805cd 100755
--- a/include/items.php
+++ b/include/items.php
@@ -334,18 +334,6 @@ function post_activity_item($arr,$allow_code = false,$deliver = true) {
 	if(! array_key_exists('mimetype',$arr))
 		$arr['mimetype'] = 'text/bbcode';
 
-	if(array_key_exists('item_private',$arr) && $arr['item_private']) {
-
-		$arr['body'] = trim(z_input_filter($arr['uid'],$arr['body'],$arr['mimetype']));
-
-		if($channel) {
-			if($channel['channel_hash'] === $arr['author_xchan']) {
-				$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
-				$arr['item_verified'] = 1;
-			}
-		}
-	}
-
 	$arr['mid']          = ((x($arr,'mid')) ? $arr['mid'] : item_message_id());
 	$arr['parent_mid']   = ((x($arr,'parent_mid')) ? $arr['parent_mid'] : $arr['mid']);
 	$arr['thr_parent']   = ((x($arr,'thr_parent')) ? $arr['thr_parent'] : $arr['mid']);
@@ -1483,35 +1471,36 @@ function item_store($arr, $allow_exec = false, $deliver = true) {
 	// obsolete, but needed so as not to throw not-null constraints on some database driveres
 	$arr['item_flags']    = ((x($arr,'item_flags'))    ? intval($arr['item_flags'])          : 0 );
 
-	// only detect language if we have text content, and if the post is private but not yet
-	// obscured, make it so.
 
-	if((! array_key_exists('item_obscured',$arr)) || $arr['item_obscured'] == 0) {
 
-		$arr['lang'] = detect_language($arr['body']);
-		// apply the input filter here - if it is obscured it has been filtered already
-		$arr['body'] = trim(z_input_filter($arr['uid'],$arr['body'],$arr['mimetype']));
+	$arr['lang'] = detect_language($arr['body']);
+	// apply the input filter here
+	$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
 
-		if(local_channel() && (local_channel() == $arr['uid']) && (! $arr['sig'])) {
+	if(local_channel() && (local_channel() == $arr['uid'])) {
+		if(! $arr['sig']) {
 			$channel = App::get_channel();
 			if($channel['channel_hash'] === $arr['author_xchan']) {
 				$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
 				$arr['item_verified'] = 1;
 			}
 		}
+	}
 
-		$allowed_languages = get_pconfig($arr['uid'],'system','allowed_languages');
+	if(! array_key_exists('sig',$arr))
+		$arr['sig'] = '';
 
-		if((is_array($allowed_languages)) && ($arr['lang']) && (! array_key_exists($arr['lang'],$allowed_languages))) {
-			$translate = array('item' => $arr, 'from' => $arr['lang'], 'to' => $allowed_languages, 'translated' => false);
-			call_hooks('item_translate', $translate);
-			if((! $translate['translated']) && (intval(get_pconfig($arr['uid'],'system','reject_disallowed_languages')))) {
-				logger('item_store: language ' . $arr['lang'] . ' not accepted for uid ' . $arr['uid']);
-				$ret['message'] = 'language not accepted';
-				return $ret;
-			}
-			$arr = $translate['item'];
+	$allowed_languages = get_pconfig($arr['uid'],'system','allowed_languages');
+
+	if((is_array($allowed_languages)) && ($arr['lang']) && (! array_key_exists($arr['lang'],$allowed_languages))) {
+		$translate = array('item' => $arr, 'from' => $arr['lang'], 'to' => $allowed_languages, 'translated' => false);
+		call_hooks('item_translate', $translate);
+		if((! $translate['translated']) && (intval(get_pconfig($arr['uid'],'system','reject_disallowed_languages')))) {
+			logger('item_store: language ' . $arr['lang'] . ' not accepted for uid ' . $arr['uid']);
+			$ret['message'] = 'language not accepted';
+			return $ret;
 		}
+		$arr = $translate['item'];
 	}
 
 	if((x($arr,'obj')) && is_array($arr['obj'])) {
@@ -1907,33 +1896,31 @@ function item_store_update($arr,$allow_exec = false, $deliver = true) {
 		return $ret;
 	}
 
-    if((! array_key_exists('item_obscured', $arr)) || $arr['item_obscured'] == 0) {
 
-		$arr['lang'] = detect_language($arr['body']);
+	$arr['lang'] = detect_language($arr['body']);
 
-        // apply the input filter here - if it is obscured it has been filtered already
-        $arr['body'] = trim(z_input_filter($arr['uid'],$arr['body'],$arr['mimetype']));
+	// apply the input filter here
+	$arr['body'] = trim($arr['body'],$arr['mimetype'],$allow_exec);
 
-		if(local_channel() && (local_channel() == $arr['uid']) && (! $arr['sig'])) {
-            $channel = App::get_channel();
-            if($channel['channel_hash'] === $arr['author_xchan']) {
-                $arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
-                $arr['item_verified'] = 1;
-            }
-        }
+	if(local_channel() && (local_channel() == $arr['uid']) && (! $arr['sig'])) {
+		$channel = App::get_channel();
+		if($channel['channel_hash'] === $arr['author_xchan']) {
+			$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
+			$arr['item_verified'] = 1;
+		}
+	}
 
-		$allowed_languages = get_pconfig($arr['uid'],'system','allowed_languages');
+	$allowed_languages = get_pconfig($arr['uid'],'system','allowed_languages');
 
-		if((is_array($allowed_languages)) && ($arr['lang']) && (! array_key_exists($arr['lang'],$allowed_languages))) {
-			$translate = array('item' => $arr, 'from' => $arr['lang'], 'to' => $allowed_languages, 'translated' => false);
-			call_hooks('item_translate', $translate);
-			if((! $translate['translated']) && (intval(get_pconfig($arr['uid'],'system','reject_disallowed_languages')))) {
-				logger('item_store: language ' . $arr['lang'] . ' not accepted for uid ' . $arr['uid']);
-				$ret['message'] = 'language not accepted';
-				return $ret;
-			}
-			$arr = $translate['item'];
+	if((is_array($allowed_languages)) && ($arr['lang']) && (! array_key_exists($arr['lang'],$allowed_languages))) {
+		$translate = array('item' => $arr, 'from' => $arr['lang'], 'to' => $allowed_languages, 'translated' => false);
+		call_hooks('item_translate', $translate);
+		if((! $translate['translated']) && (intval(get_pconfig($arr['uid'],'system','reject_disallowed_languages')))) {
+			logger('item_store: language ' . $arr['lang'] . ' not accepted for uid ' . $arr['uid']);
+			$ret['message'] = 'language not accepted';
+			return $ret;
 		}
+		$arr = $translate['item'];
 	}
 
 	if((x($arr,'obj')) && is_array($arr['obj'])) {
-- 
cgit v1.2.3


From bcd02edd8032ba3a3de40cbc35dda3529da00a7c Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Wed, 15 Mar 2017 15:53:03 -0700
Subject: ensure z_input_filter is called exactly once, since we now depend on
 the number of htmlspecialchars operations for markdown content. Also ensure
 that the content is escaped the correct number of times on imported items.

---
 include/items.php | 68 +++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 46 insertions(+), 22 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index c978805cd..ace8b2fb2 100755
--- a/include/items.php
+++ b/include/items.php
@@ -528,11 +528,6 @@ function get_item_elements($x,$allow_code = false) {
 
 	$arr = array();
 
-	if($allow_code)
-		$arr['body'] = $x['body'];
-	else
-		$arr['body'] = (($x['body']) ? htmlspecialchars($x['body'],ENT_COMPAT,'UTF-8',false) : '');
-
 	$key = get_config('system','pubkey');
 
 	$maxlen = get_max_import_size();
@@ -640,7 +635,17 @@ function get_item_elements($x,$allow_code = false) {
 			return array();
 	}
 
+	// Check signature on the body text received. 
+	// This presents an issue that we aren't verifying the text that is actually displayed
+	// on this site. We are however verifying the received text was exactly as received.
+	// We have every right to strip content that poses a security risk. You are welcome to
+	// create a plugin to verify the content after filtering if this offends you.  
+
 	if($arr['sig']) {
+
+		// check the supplied signature against the supplied content.
+		// Note that we will purify the content which could change it.
+
 		$r = q("select xchan_pubkey from xchan where xchan_hash = '%s' limit 1",
 			dbesc($arr['author_xchan'])
 		);
@@ -650,6 +655,14 @@ function get_item_elements($x,$allow_code = false) {
 			logger('get_item_elements: message verification failed.');
 	}
 
+	// if the input is markdown, remove one level of html escaping.
+	// It will be re-applied in item_store() and/or item_store_update().
+	// Do this after signature checking as the original signature
+	// was generated on the escaped content.
+
+	if($arr['mimetype'] === 'text/markdown')
+		$arr['body'] = \Zotlabs\Lib\MarkdownSoap::unescape($arr['body']);
+
 	if(array_key_exists('revision',$x)) {
 
 		// extended export encoding
@@ -1475,14 +1488,20 @@ function item_store($arr, $allow_exec = false, $deliver = true) {
 
 	$arr['lang'] = detect_language($arr['body']);
 	// apply the input filter here
-	$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
-
-	if(local_channel() && (local_channel() == $arr['uid'])) {
-		if(! $arr['sig']) {
-			$channel = App::get_channel();
-			if($channel['channel_hash'] === $arr['author_xchan']) {
-				$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
-				$arr['item_verified'] = 1;
+
+	if(array_key_exists('input_filtered_signed',$arr)) {
+		unset($arr['input_filtered_signed']);
+	}
+	else {
+		$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
+
+		if(local_channel() && (local_channel() == $arr['uid'])) {
+			if(! $arr['sig']) {
+				$channel = App::get_channel();
+				if($channel['channel_hash'] === $arr['author_xchan']) {
+					$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
+					$arr['item_verified'] = 1;
+				}
 			}
 		}
 	}
@@ -1896,17 +1915,22 @@ function item_store_update($arr,$allow_exec = false, $deliver = true) {
 		return $ret;
 	}
 
-
 	$arr['lang'] = detect_language($arr['body']);
 
-	// apply the input filter here
-	$arr['body'] = trim($arr['body'],$arr['mimetype'],$allow_exec);
-
-	if(local_channel() && (local_channel() == $arr['uid']) && (! $arr['sig'])) {
-		$channel = App::get_channel();
-		if($channel['channel_hash'] === $arr['author_xchan']) {
-			$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
-			$arr['item_verified'] = 1;
+	if(array_key_exists('input_filtered_signed',$arr)) {
+		unset($arr['input_filtered_signed']);
+	}
+	else {
+		$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
+
+		if(local_channel() && (local_channel() == $arr['uid'])) {
+			if(! $arr['sig']) {
+				$channel = App::get_channel();
+				if($channel['channel_hash'] === $arr['author_xchan']) {
+					$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
+					$arr['item_verified'] = 1;
+				}
+			}
 		}
 	}
 
-- 
cgit v1.2.3


From fb3f63bf596e77fe25c8b60c74651e39b62b1436 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Wed, 15 Mar 2017 21:49:27 -0700
Subject: input filter issue

---
 include/items.php | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index ace8b2fb2..c929bb378 100755
--- a/include/items.php
+++ b/include/items.php
@@ -532,6 +532,8 @@ function get_item_elements($x,$allow_code = false) {
 
 	$maxlen = get_max_import_size();
 
+	$arr['body'] = $x['body'];
+
 	if($maxlen && mb_strlen($arr['body']) > $maxlen) {
 		$arr['body'] = mb_substr($arr['body'],0,$maxlen,'UTF-8');
 		logger('get_item_elements: message length exceeds max_import_size: truncated');
-- 
cgit v1.2.3


From 051759580dc7aff7d5b349b3b5271f054c7d3423 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 16 Mar 2017 01:16:29 -0700
Subject: namespace error

---
 include/items.php | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index c929bb378..c6aedc2e4 100755
--- a/include/items.php
+++ b/include/items.php
@@ -528,12 +528,10 @@ function get_item_elements($x,$allow_code = false) {
 
 	$arr = array();
 
-	$key = get_config('system','pubkey');
+	$arr['body'] = $x['body'];
 
 	$maxlen = get_max_import_size();
 
-	$arr['body'] = $x['body'];
-
 	if($maxlen && mb_strlen($arr['body']) > $maxlen) {
 		$arr['body'] = mb_substr($arr['body'],0,$maxlen,'UTF-8');
 		logger('get_item_elements: message length exceeds max_import_size: truncated');
-- 
cgit v1.2.3


From cf11ba1147355f0583016db2cec4f7793e656dfc Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 16 Mar 2017 18:19:03 -0700
Subject: simplify the message signing spaghetti

---
 include/items.php | 61 ++++++++++++++++++++++++++-----------------------------
 1 file changed, 29 insertions(+), 32 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index c6aedc2e4..3d0f380b4 100755
--- a/include/items.php
+++ b/include/items.php
@@ -1408,6 +1408,26 @@ function get_profile_elements($x) {
 }
 
 
+
+
+function item_sign(&$item) {
+
+	if(array_key_exists('sig',$item) && $item['sig'])
+		return;
+
+	$r = q("select channel_prvkey from channel where channel_id = %d and channel_hash = '%s' ",
+			intval($item['uid']),
+			dbesc($item['author_xchan'])
+	);
+	if(! $r)
+		return;
+
+	$item['sig'] = base64url_encode(rsa_sign($item['body'],$r[0]['channel_prvkey']));
+	$item['item_verified'] = 1;
+
+}
+
+
 /**
  * @brief
  *
@@ -1487,24 +1507,12 @@ function item_store($arr, $allow_exec = false, $deliver = true) {
 
 
 	$arr['lang'] = detect_language($arr['body']);
+
 	// apply the input filter here
 
-	if(array_key_exists('input_filtered_signed',$arr)) {
-		unset($arr['input_filtered_signed']);
-	}
-	else {
-		$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
-
-		if(local_channel() && (local_channel() == $arr['uid'])) {
-			if(! $arr['sig']) {
-				$channel = App::get_channel();
-				if($channel['channel_hash'] === $arr['author_xchan']) {
-					$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
-					$arr['item_verified'] = 1;
-				}
-			}
-		}
-	}
+	$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
+
+	item_sign($arr);
 
 	if(! array_key_exists('sig',$arr))
 		$arr['sig'] = '';
@@ -1917,22 +1925,11 @@ function item_store_update($arr,$allow_exec = false, $deliver = true) {
 
 	$arr['lang'] = detect_language($arr['body']);
 
-	if(array_key_exists('input_filtered_signed',$arr)) {
-		unset($arr['input_filtered_signed']);
-	}
-	else {
-		$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
-
-		if(local_channel() && (local_channel() == $arr['uid'])) {
-			if(! $arr['sig']) {
-				$channel = App::get_channel();
-				if($channel['channel_hash'] === $arr['author_xchan']) {
-					$arr['sig'] = base64url_encode(rsa_sign($arr['body'],$channel['channel_prvkey']));
-					$arr['item_verified'] = 1;
-				}
-			}
-		}
-	}
+	// apply the input filter here
+
+	$arr['body'] = trim(z_input_filter($arr['body'],$arr['mimetype'],$allow_exec));
+
+	item_sign($arr);
 
 	$allowed_languages = get_pconfig($arr['uid'],'system','allowed_languages');
 
-- 
cgit v1.2.3


From d985db60cca1256698f15a9cb7a39688c72ec082 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Sun, 2 Apr 2017 17:09:51 -0700
Subject: import authors from any unrecognised network as network 'unknown'.

---
 include/items.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 3d0f380b4..4ef52fb41 100755
--- a/include/items.php
+++ b/include/items.php
@@ -762,6 +762,8 @@ function import_author_xchan($x) {
 	if($arr['xchan_hash'])
 		return $arr['xchan_hash'];
 
+	$y = false;
+
 	if((! array_key_exists('network', $x)) || ($x['network'] === 'zot')) {
 		$y = import_author_zot($x);
 	}
@@ -770,9 +772,7 @@ function import_author_xchan($x) {
 		$y = import_author_rss($x);
 	}
 
-	if($x['network'] === 'unknown') {
-		$y = import_author_unknown($x);
-	}
+	$y = import_author_unknown($x);
 
 	return(($y) ? $y : false);
 }
-- 
cgit v1.2.3


From 1a28fb2a0c8b183fc4463c7f2c08212d068deb69 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Sun, 2 Apr 2017 17:12:42 -0700
Subject: logic error

---
 include/items.php | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 4ef52fb41..ea72d1d4e 100755
--- a/include/items.php
+++ b/include/items.php
@@ -772,9 +772,11 @@ function import_author_xchan($x) {
 		$y = import_author_rss($x);
 	}
 
-	$y = import_author_unknown($x);
+	if(! $y) {
+		$y = import_author_unknown($x);
+	}
 
-	return(($y) ? $y : false);
+	return($y);
 }
 
 /**
-- 
cgit v1.2.3


From 26125bcf0b8ce98fa4522f89792d7e77f25ff1ca Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Sun, 2 Apr 2017 22:20:37 -0700
Subject: sql error photos_albums_list with non-logged-in viewer

---
 include/items.php | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index ea72d1d4e..e3147dea4 100755
--- a/include/items.php
+++ b/include/items.php
@@ -1776,9 +1776,12 @@ logger('revision: ' . $arr['revision']);
 		intval($arr['revision'])
 	);
 
-	if($r && count($r)) {
+	if($r) {
+		// This will gives us a fresh copy of what's now in the DB and undo the db escaping, 
+		// which really messes up the notifications
+
 		$current_post = $r[0]['id'];
-		$arr = $r[0];  // This will gives us a fresh copy of what's now in the DB and undo the db escaping, which really messes up the notifications
+		$arr = $r[0];
 		logger('item_store: created item ' . $current_post, LOGGER_DEBUG);
 	}
 	else {
-- 
cgit v1.2.3


From 063577c19ea45918e332ec640fb7369bf757b348 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Sun, 9 Apr 2017 20:25:41 -0700
Subject: some signatures showing as invalid because of recent
 import_author_zot() changes.

---
 include/items.php | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index c3b57f6cb..f290b5ec1 100755
--- a/include/items.php
+++ b/include/items.php
@@ -625,14 +625,17 @@ function get_item_elements($x,$allow_code = false) {
 		return array();
 
 	// save a potentially expensive lookup if author == owner
+
 	if($arr['author_xchan'] === make_xchan_hash($x['owner']['guid'],$x['owner']['guid_sig']))
 		$arr['owner_xchan'] = $arr['author_xchan'];
 	else {
 		$xchan_hash = import_author_xchan($x['owner']);
-		if($xchan_hash)
+		if($xchan_hash) {
 			$arr['owner_xchan'] = $xchan_hash;
-		else
+		}
+		else {
 			return array();
+		}
 	}
 
 	// Check signature on the body text received. 
@@ -649,10 +652,25 @@ function get_item_elements($x,$allow_code = false) {
 		$r = q("select xchan_pubkey from xchan where xchan_hash = '%s' limit 1",
 			dbesc($arr['author_xchan'])
 		);
-		if($r && rsa_verify($x['body'],base64url_decode($arr['sig']),$r[0]['xchan_pubkey']))
-			$arr['item_verified'] = 1;
-		else
-			logger('get_item_elements: message verification failed.');
+		if($r) {
+			if($r[0]['xchan_pubkey']) {
+				if(rsa_verify($x['body'],base64url_decode($arr['sig']),$r[0]['xchan_pubkey'])) {
+					$arr['item_verified'] = 1;
+				}
+				else {
+					logger('get_item_elements: message verification failed.');
+				}
+			}
+			else {
+
+				// If we don't have a public key, strip the signature so it won't show as invalid.
+				// This won't happen in normal use, but could happen if import_author_xchan()
+				// failed to load the zot-info packet due to a server failure and had 
+				// to create an alternate xchan with network 'unknown'
+
+				unset($arr['sig']);
+			}
+		}
 	}
 
 	// if the input is markdown, remove one level of html escaping.
@@ -1107,7 +1125,7 @@ function encode_item_xchan($xchan) {
 	$ret['address']  = $xchan['xchan_addr'];
 	$ret['url']      = $xchan['xchan_url'];
 	$ret['network']  = $xchan['xchan_network'];
-	$ret['photo']    = array('mimetype' => $xchan['xchan_photo_mimetype'], 'src' => $xchan['xchan_photo_m']);
+	$ret['photo']    = [ 'mimetype' => $xchan['xchan_photo_mimetype'], 'src' => $xchan['xchan_photo_m'] ];
 	$ret['guid']     = $xchan['xchan_guid'];
 	$ret['guid_sig'] = $xchan['xchan_guid_sig'];
 
-- 
cgit v1.2.3


From 94f15e355166f0abd3698414948abff2397a7ad0 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 20 Apr 2017 20:19:15 -0700
Subject: The rest of the library and backend changes to support client-side
 e2ee and deprecate previous uses of item_obscured.

---
 include/items.php | 35 +++++++++--------------------------
 1 file changed, 9 insertions(+), 26 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index f290b5ec1..c71e8011d 100755
--- a/include/items.php
+++ b/include/items.php
@@ -605,6 +605,9 @@ function get_item_elements($x,$allow_code = false) {
 		if(in_array('notshown',$x['flags']))
 			$arr['item_notshown'] = 1;
 
+		if(in_array('obscured',$x['flags']))
+			$arr['item_obscured'] = 1;
+
 		// hidden item are no longer propagated - notshown may be a suitable alternative
 
 		if(in_array('hidden',$x['flags']))
@@ -707,7 +710,7 @@ function get_item_elements($x,$allow_code = false) {
 		// local only		$arr['item_relay'] = $x['item_relay'];
 		$arr['item_mentionsme'] = $x['item_mentionsme'];
 		$arr['item_nocomment'] = $x['item_nocomment'];
-		// local only $arr['item_obscured'] = $x['item_obscured'];
+		$arr['item_obscured'] = $x['item_obscured'];
 		// local only $arr['item_verified'] = $x['item_verified'];
 		$arr['item_retained'] = $x['item_retained'];
 		$arr['item_rss'] = $x['item_rss'];
@@ -955,13 +958,6 @@ function encode_item($item,$mirror = false) {
 
 	$key = get_config('system','prvkey');
 
-	if(array_key_exists('item_obscured',$item) && intval($item['item_obscured'])) {
-		if($item['title'])
-			$item['title'] = crypto_unencapsulate(json_decode($item['title'],true),$key);
-		if($item['body'])
-			$item['body'] = crypto_unencapsulate(json_decode($item['body'],true),$key);
-	}
-
 	// If we're trying to backup an item so that it's recoverable or for export/imprt,
 	// add all the attributes we need to recover it
 
@@ -1309,7 +1305,9 @@ function encode_item_flags($item) {
 		$ret[] = 'nsfw';
 	if(intval($item['item_consensus']))
 		$ret[] = 'consensus';
-	if(intval($item['item_private']))
+	if(intval($item['item_obscured']))
+		$ret[] = 'obscured';
+	if(intval($item['item_privat']))
 		$ret[] = 'private';
 
 	return $ret;
@@ -2479,15 +2477,7 @@ function tag_deliver($uid, $item_id) {
 		// Now let's check if this mention was inside a reshare so we don't spam a forum
 		// If it's private we may have to unobscure it momentarily so that we can parse it.
 
-		$body = '';
-
-		if(intval($item['item_obscured'])) {
-			$key = get_config('system','prvkey');
-			if($item['body'])
-				$body = crypto_unencapsulate(json_decode($item['body'],true),$key);
-		}
-		else
-			$body = $item['body'];
+		$body = $item['body'];
 
 		$body = preg_replace('/\[share(.*?)\[\/share\]/','',$body);
 
@@ -2638,11 +2628,6 @@ function tgroup_check($uid,$item) {
 
 	$body = $item['body'];
 
-	if(array_key_exists('item_obscured',$item) && intval($item['item_obscured']) && $body) {
-		$key = get_config('system','prvkey');
-		$body = crypto_unencapsulate(json_decode($body,true),$key);
-	}
-
 	$body = preg_replace('/\[share(.*?)\[\/share\]/','',$body);
 
 //	$pattern = '/@\!?\[zrl\=' . preg_quote($term['url'],'/') . '\]' . preg_quote($term['term'] . '+','/') . '\[\/zrl\]/';
@@ -2736,7 +2721,6 @@ function start_delivery_chain($channel, $item, $item_id, $parent) {
 	$item_origin = 1;
 	$item_uplink = 0;
 	$item_nocomment = 0;
-	$item_obscured = 0;
 
 	$flag_bits = $item['item_flags'];
 
@@ -2759,11 +2743,10 @@ function start_delivery_chain($channel, $item, $item_id, $parent) {
 	$title = $item['title'];
 	$body  = $item['body'];
 
-	$r = q("update item set item_uplink = %d, item_nocomment = %d, item_obscured = %d, item_flags = %d, owner_xchan = '%s', allow_cid = '%s', allow_gid = '%s',
+	$r = q("update item set item_uplink = %d, item_nocomment = %d, item_flags = %d, owner_xchan = '%s', allow_cid = '%s', allow_gid = '%s',
 		deny_cid = '%s', deny_gid = '%s', item_private = %d, public_policy = '%s', comment_policy = '%s', title = '%s', body = '%s', item_wall = %d, item_origin = %d  where id = %d",
 		intval($item_uplink),
 		intval($item_nocomment),
-		intval($item_obscured),
 		intval($flag_bits),
 		dbesc($channel['channel_hash']),
 		dbesc($channel['channel_allow_cid']),
-- 
cgit v1.2.3


From f1c6fd08c462b663a26cf58cfd81591129057c41 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Sun, 23 Apr 2017 20:28:04 -0700
Subject: database upgrade

---
 include/items.php | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index c71e8011d..a4862208e 100755
--- a/include/items.php
+++ b/include/items.php
@@ -4263,10 +4263,7 @@ function sync_an_item($channel_id,$item_id) {
 	if($r) {
 		xchan_query($r);
 		$sync_item = fetch_post_tags($r);
-		$rid = q("select * from item_id where iid = %d",
-			intval($item_id)
-		);
-		build_sync_packet($channel_d,array('item' => array(encode_item($sync_item[0],true)),'item_id' => $rid));
+		build_sync_packet($channel_d,array('item' => array(encode_item($sync_item[0],true))));
 	}
 }
 
-- 
cgit v1.2.3


From b88be137ca68c6b52e13ce1738fbc55651f37138 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Tue, 25 Apr 2017 20:48:38 -0700
Subject: provide a broken feed for services which require a broken feed

---
 include/items.php | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index a4862208e..1f79ee567 100755
--- a/include/items.php
+++ b/include/items.php
@@ -3885,6 +3885,10 @@ function items_fetch($arr,$channel = null,$observer_hash = null,$client_mode = C
 		}
 	}
 
+	if(intval($arr['compat']) === 1) {
+		$sql_extra = " AND author_xchan = owner_xchan and item_wall = 1 and item_private = 0 ";
+	}
+
 	if ($arr['datequery']) {
 		$sql_extra3 .= protect_sprintf(sprintf(" AND item.created <= '%s' ", dbesc(datetime_convert('UTC','UTC',$arr['datequery']))));
 	}
-- 
cgit v1.2.3


From e8ea8d203db5bdf62995f1831b1503ac11556435 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Wed, 26 Apr 2017 23:05:30 -0700
Subject: more client-side mail privacy work

---
 include/items.php | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 1f79ee567..7cad3b168 100755
--- a/include/items.php
+++ b/include/items.php
@@ -1333,6 +1333,8 @@ function encode_mail($item,$extended = false) {
 	$x['body']           = $item['body'];
 	$x['from']           = encode_item_xchan($item['from']);
 	$x['to']             = encode_item_xchan($item['to']);
+	$x['raw']            = $item['mail_raw'];
+	$x['mimetype']       = $item['mail_mimetype'];
 
 	if($item['attach'])
 		$x['attach']     = json_decode($item['attach'],true);
@@ -1366,9 +1368,16 @@ function get_mail_elements($x) {
 
 	$arr = array();
 
-	$arr['body']         = (($x['body']) ? htmlspecialchars($x['body'], ENT_COMPAT,'UTF-8',false) : '');
-	$arr['title']        = (($x['title'])? htmlspecialchars($x['title'],ENT_COMPAT,'UTF-8',false) : '');
+	if(intval($x['raw'])) {
+		$arr['mail_raw'] = intval($x['raw']);
+		$arr['body']     = $x['body'];
+	}
+	else {
+		$arr['body']         = (($x['body']) ? htmlspecialchars($x['body'], ENT_COMPAT,'UTF-8',false) : '');
+	}
 
+	$arr['title']        = (($x['title'])? htmlspecialchars($x['title'],ENT_COMPAT,'UTF-8',false) : '');
+	$arr['mail_mimetype'] = (($x['mimetype']) ? htmlspecialchars($x['mimetype'],ENT_COMPAT,'UTF-8',false) : 'text/bbcode');
 	$arr['conv_guid']    = (($x['conv_guid'])? htmlspecialchars($x['conv_guid'],ENT_COMPAT,'UTF-8',false) : '');
 
 	$arr['created']      = datetime_convert('UTC','UTC',$x['created']);
@@ -2949,8 +2958,10 @@ function mail_store($arr) {
 	$arr['parent_mid']    = ((x($arr,'parent_mid'))    ? notags(trim($arr['parent_mid']))    : '');
 	$arr['body']          = ((x($arr,'body'))          ? trim($arr['body'])                  : '');
 	$arr['conv_guid']     = ((x($arr,'conv_guid'))     ? trim($arr['conv_guid'])             : '');
+	$arr['mail_mimetype'] = ((x($arr,'mail_mimetype')) ? trim($arr['mail_mimetype'])         : 'text/bbcode');
 
 	$arr['mail_flags']    = ((x($arr,'mail_flags'))    ? intval($arr['mail_flags'])          : 0 );
+	$arr['mail_raw']      = ((x($arr,'mail_raw'))      ? intval($arr['mail_raw'])            : 0 );
 
 	if(! $arr['parent_mid']) {
 		logger('mail_store: missing parent');
-- 
cgit v1.2.3


From 89e7e5e4fbafa9bac7263513315f89cfe7ac02fb Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 27 Apr 2017 20:08:38 -0700
Subject: make mail_store() safe for postgres or mysql5.7 - although this
 interface needs to be refactored in the future

---
 include/items.php | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 7cad3b168..30aa30048 100755
--- a/include/items.php
+++ b/include/items.php
@@ -2945,8 +2945,14 @@ function mail_store($arr) {
 			$arr['body'] = escape_tags($arr['body']);
 	}
 
-	if(array_key_exists('attach',$arr) && is_array($arr['attach']))
-		$arr['attach'] = json_encode($arr['attach']);
+	if(array_key_exists('attach',$arr)) {
+		if(is_array($arr['attach'])) {
+			$arr['attach'] = json_encode($arr['attach']);
+		}
+	}
+	else {
+		$arr['attach'] = '';
+	}
 
 	$arr['account_id']    = ((x($arr,'account_id'))           ? intval($arr['account_id'])                 : 0);
 	$arr['mid']           = ((x($arr,'mid'))           ? notags(trim($arr['mid']))           : random_string());
@@ -2957,6 +2963,7 @@ function mail_store($arr) {
 	$arr['title']         = ((x($arr,'title'))         ? trim($arr['title'])         : '');
 	$arr['parent_mid']    = ((x($arr,'parent_mid'))    ? notags(trim($arr['parent_mid']))    : '');
 	$arr['body']          = ((x($arr,'body'))          ? trim($arr['body'])                  : '');
+	$arr['sig']           = ((x($arr,'sig'))           ? trim($arr['sig'])                   : '');
 	$arr['conv_guid']     = ((x($arr,'conv_guid'))     ? trim($arr['conv_guid'])             : '');
 	$arr['mail_mimetype'] = ((x($arr,'mail_mimetype')) ? trim($arr['mail_mimetype'])         : 'text/bbcode');
 
-- 
cgit v1.2.3


From b2d2dcc7fe7af1b61e2636ed7e12c805fe67f49b Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Tue, 2 May 2017 20:17:47 -0700
Subject: more code refactoring to put external protocol dependencies in
 plugins.

---
 include/items.php | 36 +++++-------------------------------
 1 file changed, 5 insertions(+), 31 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 30aa30048..a3f423836 100755
--- a/include/items.php
+++ b/include/items.php
@@ -801,37 +801,6 @@ function import_author_xchan($x) {
 
 }
 
-/**
- * @brief Imports an author from Diaspora.
- *
- * @param array $x an associative array with
- *   * \e string \b address
- * @return boolean|string false on error, otherwise xchan_hash of the new entry
- */
-function import_author_diaspora($x) {
-	if(! $x['address'])
-		return false;
-
-	$r = q("select * from xchan where xchan_addr = '%s' limit 1",
-		dbesc($x['address'])
-	);
-	if($r) {
-		logger('in_cache: ' . $x['address'], LOGGER_DATA);
-		return $r[0]['xchan_hash'];
-	}
-
-	if(discover_by_webbie($x['address'])) {
-		$r = q("select xchan_hash from xchan where xchan_addr = '%s' limit 1",
-			dbesc($x['address'])
-		);
-		if($r)
-			return $r[0]['xchan_hash'];
-	}
-
-	return false;
-
-}
-
 /**
  * @brief Imports an author from a RSS feed.
  *
@@ -889,6 +858,11 @@ function import_author_rss($x) {
 
 function import_author_unknown($x) {
 
+	$arr = [ 'author' => $x, 'result' => false ];
+	call_hooks('import_author', $arr);
+	if($arr['result'])
+		return $arr['result'];
+
 	if(! $x['url'])
 		return false;
 
-- 
cgit v1.2.3


From eeb9103e267a01ad5f076afeb02bedabfcc70db9 Mon Sep 17 00:00:00 2001
From: Klaus Weidenbach <Klaus.Weidenbach@gmx.net>
Date: Fri, 5 May 2017 00:23:57 +0200
Subject: :bulb: correct some Doxygen syntax mistakes.

---
 include/items.php | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index c895a6d6e..4b90840e2 100755
--- a/include/items.php
+++ b/include/items.php
@@ -298,11 +298,13 @@ function add_source_route($iid, $hash) {
  * or other processing is performed.
  *
  * @param array $arr
+ * @param boolean $allow_code (optional) default false
+ * @param boolean $deliver (optional) default true
  * @returns array
  *  * \e boolean \b success true or false
  *  * \e array \b activity the resulting activity if successful
  */
-function post_activity_item($arr,$allow_code = false,$deliver = true) {
+function post_activity_item($arr, $allow_code = false, $deliver = true) {
 
 	$ret = array('success' => false);
 
@@ -348,7 +350,7 @@ function post_activity_item($arr,$allow_code = false,$deliver = true) {
 	if(($is_comment) && ($arr['obj_type'] === ACTIVITY_OBJ_NOTE))
 		$arr['obj_type'] = ACTIVITY_OBJ_COMMENT;
 
-	if(! ( array_key_exists('allow_cid',$arr) || array_key_exists('allow_gid',$arr) 
+	if(! ( array_key_exists('allow_cid',$arr) || array_key_exists('allow_gid',$arr)
 		|| array_key_exists('deny_cid',$arr) || array_key_exists('deny_gid',$arr))) {
 		$arr['allow_cid']    = $channel['channel_allow_cid'];
 		$arr['allow_gid']    = $channel['channel_allow_gid'];
@@ -648,11 +650,11 @@ function get_item_elements($x,$allow_code = false) {
 		}
 	}
 
-	// Check signature on the body text received. 
+	// Check signature on the body text received.
 	// This presents an issue that we aren't verifying the text that is actually displayed
 	// on this site. We are however verifying the received text was exactly as received.
 	// We have every right to strip content that poses a security risk. You are welcome to
-	// create a plugin to verify the content after filtering if this offends you.  
+	// create a plugin to verify the content after filtering if this offends you.
 
 	if($arr['sig']) {
 
@@ -675,7 +677,7 @@ function get_item_elements($x,$allow_code = false) {
 
 				// If we don't have a public key, strip the signature so it won't show as invalid.
 				// This won't happen in normal use, but could happen if import_author_xchan()
-				// failed to load the zot-info packet due to a server failure and had 
+				// failed to load the zot-info packet due to a server failure and had
 				// to create an alternate xchan with network 'unknown'
 
 				unset($arr['sig']);
@@ -1518,7 +1520,7 @@ function item_store($arr, $allow_exec = false, $deliver = true) {
 	if(array_key_exists('cancel',$arr) && $arr['cancel']) {
 		logger('cancelled by plugin');
 		return $ret;
-	}		
+	}
 
 	if(! $arr['uid']) {
 		logger('item_store: no uid');
@@ -1843,7 +1845,7 @@ logger('revision: ' . $arr['revision']);
 	);
 
 	if($r) {
-		// This will gives us a fresh copy of what's now in the DB and undo the db escaping, 
+		// This will gives us a fresh copy of what's now in the DB and undo the db escaping,
 		// which really messes up the notifications
 
 		$current_post = $r[0]['id'];
@@ -1906,7 +1908,7 @@ logger('revision: ' . $arr['revision']);
 
 	// update the commented timestamp on the parent - unless this is potentially a clone of an older item
 	// which we don't wish to bring to the surface. As the queue only holds deliveries for 3 days, it's
-	// suspected of being an older cloned item if the creation time is older than that. 
+	// suspected of being an older cloned item if the creation time is older than that.
 
 	if($arr['created'] > datetime_convert('','','now - 4 days')) {
 		$z = q("select max(created) as commented from item where parent_mid = '%s' and uid = %d and item_delayed = 0 ",
@@ -1952,7 +1954,7 @@ function item_store_update($arr,$allow_exec = false, $deliver = true) {
 	if(array_key_exists('cancel',$arr) && $arr['cancel']) {
 		logger('cancelled by plugin');
 		return $ret;
-	}		
+	}
 
 	if(! intval($arr['uid'])) {
 		logger('item_store_update: no uid');
@@ -4502,12 +4504,12 @@ function item_create_edit_activity($post) {
 	$new_item['id'] = 0;
 	$new_item['parent'] = 0;
 	$new_item['mid'] = item_message_id();
-	
+
 	$new_item['body'] = sprintf( t('[Edited %s]'), (($update_item['item_thread_top']) ? t('Post','edit_activity') : t('Comment','edit_activity')));
 
 	$new_item['body'] .= "\n\n";
 	$new_item['body'] .= $update_item['body'];
-	
+
 	$new_item['sig'] = '';
 
 	$new_item['verb'] = ACTIVITY_UPDATE;
@@ -4533,10 +4535,10 @@ function item_create_edit_activity($post) {
 				array('rel' => 'photo', 'type' => $item_author['xchan_photo_mimetype'], 'href' => $item_author['xchan_photo_m'])),
 			),
 	));
-	
 
 
-	$x = post_activity_item($new_item);	
+
+	$x = post_activity_item($new_item);
 
 	$post_id = $x['id'];
 	if($post_id) {
@@ -4551,5 +4553,5 @@ function item_create_edit_activity($post) {
 	}
 
 	\Zotlabs\Daemon\Master::Summon(array('Notifier', 'edit_activity', $post_id));
-	
+
 }
-- 
cgit v1.2.3


From faf98147882857f09ed5bdb694bebaae416b2526 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 4 May 2017 19:18:54 -0700
Subject: typo

---
 include/items.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index a3f423836..7b88ee7f3 100755
--- a/include/items.php
+++ b/include/items.php
@@ -1281,7 +1281,7 @@ function encode_item_flags($item) {
 		$ret[] = 'consensus';
 	if(intval($item['item_obscured']))
 		$ret[] = 'obscured';
-	if(intval($item['item_privat']))
+	if(intval($item['item_private']))
 		$ret[] = 'private';
 
 	return $ret;
-- 
cgit v1.2.3


From 977677d0051db24c6b7666a73c4dd6a48fe89377 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 4 May 2017 19:18:54 -0700
Subject: typo

---
 include/items.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index c895a6d6e..ffb02a8f0 100755
--- a/include/items.php
+++ b/include/items.php
@@ -1323,7 +1323,7 @@ function encode_item_flags($item) {
 		$ret[] = 'consensus';
 	if(intval($item['item_obscured']))
 		$ret[] = 'obscured';
-	if(intval($item['item_privat']))
+	if(intval($item['item_private']))
 		$ret[] = 'private';
 
 	return $ret;
-- 
cgit v1.2.3


From 893ae17b99d1a1a720a58879e8075fc9bb1de087 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Tue, 30 May 2017 17:06:50 -0700
Subject: provide ability to search webpages (either public or which contain
 the observer in the acl) in addition to conversations

---
 include/items.php | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index a375ee3c3..707f07f4a 100755
--- a/include/items.php
+++ b/include/items.php
@@ -175,6 +175,13 @@ function item_normal() {
 		and item.item_blocked = 0 ";
 }
 
+function item_normal_search() {
+	return " and item.item_hidden = 0 and item.item_type in (0,3) and item.item_deleted = 0
+		and item.item_unpublished = 0 and item.item_delayed = 0 and item.item_pending_remove = 0
+		and item.item_blocked = 0 ";
+}
+
+
 /**
  * @brief
  *
-- 
cgit v1.2.3


From 5b3e863323a6b70651454c0972a586e677f55c3d Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Mon, 12 Jun 2017 16:17:23 -0700
Subject: add 'can_comment_on_post' hook so we can better deal with the
 complications of Diaspora policy

---
 include/items.php | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 707f07f4a..e79707aeb 100755
--- a/include/items.php
+++ b/include/items.php
@@ -219,6 +219,11 @@ function can_comment_on_post($observer_xchan, $item) {
 
 //	logger('can_comment_on_post: comment_policy: ' . $item['comment_policy'], LOGGER_DEBUG);
 
+	$x = [ 'observer_hash' => $observer_xchan, 'item' => $item, 'allowed' => 'unset' ];
+	call_hooks('can_comment_on_post',$x);
+	if($x['allowed'] !== 'unset')
+		return $x['allowed'];
+
 	if(! $observer_xchan)
 		return false;
 
-- 
cgit v1.2.3


From b917cf1eccc62f5f533c0a61ffd699764b39e404 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Sun, 18 Jun 2017 22:25:41 -0700
Subject: allow moderated comments like wordpress if permissions are compatible

---
 include/items.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index e79707aeb..9e1d5c1ba 100755
--- a/include/items.php
+++ b/include/items.php
@@ -906,7 +906,7 @@ function import_author_unknown($x) {
 		$photos = import_xchan_photo($x['photo']['src'],$x['url']);
 
 		if($photos) {
-			$r = q("update xchan set xchan_photo_date = '%s', xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s' where xchan_url = '%s' and xchan_network = 'unknown'",
+			$r = q("update xchan set xchan_photo_date = '%s', xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s' where xchan_hash = '%s' and xchan_network = 'unknown'",
 				dbesc(datetime_convert()),
 				dbesc($photos[0]),
 				dbesc($photos[1]),
-- 
cgit v1.2.3


From 0233e9ad2a8c210f29c553a791b85d52cd6d5d85 Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Tue, 20 Jun 2017 16:33:53 -0700
Subject: still an issue with caching import_author_zot

---
 include/items.php | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 9e1d5c1ba..126f0a174 100755
--- a/include/items.php
+++ b/include/items.php
@@ -803,6 +803,10 @@ function import_author_xchan($x) {
 		$y = import_author_zot($x);
 	}
 
+	// if we were told that it's a zot connection, don't probe/import anything else
+	if(array_key_exists('network',$x) && $x['network'] === 'zot')
+		return $y;
+
 	if($x['network'] === 'rss') {
 		$y = import_author_rss($x);
 	}
-- 
cgit v1.2.3


From 3b4f70ae83e458720f7af2446c66dcf5e0d55ace Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Thu, 22 Jun 2017 17:31:08 -0700
Subject: Several things about mail storage weren't quite right.

---
 include/items.php | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 126f0a174..b313193fb 100755
--- a/include/items.php
+++ b/include/items.php
@@ -8,6 +8,7 @@ use Zotlabs\Lib as Zlib;
 require_once('include/bbcode.php');
 require_once('include/oembed.php');
 require_once('include/crypto.php');
+require_once('include/message.php');
 require_once('include/feedutils.php');
 require_once('include/photo/photo_driver.php');
 require_once('include/permissions.php');
@@ -2932,6 +2933,8 @@ function mail_store($arr) {
 		return 0;
 	}
 
+	$channel = channelx_by_n($arr['channel_id']);
+
 	if(! $arr['mail_obscured']) {
 		if((strpos($arr['body'],'<') !== false) || (strpos($arr['body'],'>') !== false))
 			$arr['body'] = escape_tags($arr['body']);
@@ -2962,11 +2965,34 @@ function mail_store($arr) {
 	$arr['mail_flags']    = ((x($arr,'mail_flags'))    ? intval($arr['mail_flags'])          : 0 );
 	$arr['mail_raw']      = ((x($arr,'mail_raw'))      ? intval($arr['mail_raw'])            : 0 );
 
-	if(! $arr['parent_mid']) {
+	
+
+	if($arr['parent_mid']) {
+		$parent_item = q("select * from mail where mid = '%s' and channel_id = %d limit 1",
+			dbesc($arr['parent_mid']),
+			intval($arr['channel_id'])
+		);
+		if(($parent_item) && (! $arr['conv_guid'])) {
+			$arr['conv_guid'] = $parent_item[0]['conv_guid']; 
+		}
+	}
+	else {
 		logger('mail_store: missing parent');
 		$arr['parent_mid'] = $arr['mid'];
 	}
 
+	if($arr['from_xchan'] === $channel['channel_hash'])
+		$conversant = $arr['to_xchan'];
+	else
+		$conversant = $arr['from_xchan'];
+
+
+	if(! $arr['conv_guid']) {
+		$x = create_conversation($channel,$conversant,(($arr['title']) ? base64url_decode(str_rot47($arr['title'])) : ''));
+		$arr['conv_guid'] = (($x) ? $x['guid'] : '');
+	}
+
+
 	$r = q("SELECT id FROM mail WHERE mid = '%s' AND channel_id = %d LIMIT 1",
 		dbesc($arr['mid']),
 		intval($arr['channel_id'])
@@ -3031,6 +3057,14 @@ function mail_store($arr) {
 		Zlib\Enotify::submit($notif_params);
 	}
 
+	if($arr['conv_guid']) {
+		$c = q("update conv set updated = '%s' where guid = '%s' and uid = %d",
+			dbesc(datetime_convert()),
+			dbesc($arr['conv_guid']),
+			intval($arr['channel_id'])
+		);
+	}
+
 	call_hooks('post_mail_end',$arr);
 	return $current_post;
 }
-- 
cgit v1.2.3


From 2d63bbb91e97d3a54440564620ef3093ecbe71fb Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Mon, 10 Jul 2017 20:18:33 -0700
Subject: prevent expiration of conversations you are involved with - allows
 you to find your own comments months from now

---
 include/items.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index b313193fb..386c25922 100755
--- a/include/items.php
+++ b/include/items.php
@@ -1737,7 +1737,7 @@ logger('revision: ' . $arr['revision']);
 
 			if($r[0]['owner_xchan'] !== $arr['owner_xchan']) {
 				$arr['owner_xchan'] = $r[0]['owner_xchan'];
-//				$uplinked_comment = true;
+				//				$uplinked_comment = true;
 			}
 
 			// if the parent is private, force privacy for the entire conversation
-- 
cgit v1.2.3


From 037e1edb1fd2e56f776d0f86f0023141949154cf Mon Sep 17 00:00:00 2001
From: zotlabs <mike@macgirvin.com>
Date: Sun, 6 Aug 2017 19:33:51 -0700
Subject: ostatus feed was missing comments

---
 include/items.php | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

(limited to 'include/items.php')

diff --git a/include/items.php b/include/items.php
index 386c25922..5a0ca01c6 100755
--- a/include/items.php
+++ b/include/items.php
@@ -3929,8 +3929,8 @@ function items_fetch($arr,$channel = null,$observer_hash = null,$client_mode = C
 		}
 	}
 
-	if(intval($arr['compat']) === 1) {
-		$sql_extra = " AND author_xchan = owner_xchan and item_wall = 1 and item_private = 0 ";
+	if($channel && intval($arr['compat']) === 1) {
+		$sql_extra = " AND author_xchan = '" . $channel['channel_hash'] . "' and item_private = 0 ";
 	}
 
 	if ($arr['datequery']) {
@@ -3940,11 +3940,6 @@ function items_fetch($arr,$channel = null,$observer_hash = null,$client_mode = C
 		$sql_extra3 .= protect_sprintf(sprintf(" AND item.created >= '%s' ", dbesc(datetime_convert('UTC','UTC',$arr['datequery2']))));
 	}
 
-	if(! array_key_exists('nouveau',$arr)) {
-		$sql_extra2 = " AND item.parent = item.id ";
-//		$sql_extra3 = '';
-	}
-
 	if($arr['search']) {
 
         if(strpos($arr['search'],'#') === 0)
@@ -4016,7 +4011,8 @@ function items_fetch($arr,$channel = null,$observer_hash = null,$client_mode = C
 	if($arr['item_type'] === '*')
 		$item_restrict = '';
 
-	if ($arr['nouveau'] && ($client_mode & CLIENT_MODE_LOAD) && $channel) {
+	if ((($arr['compat']) || ($arr['nouveau'] && ($client_mode & CLIENT_MODE_LOAD))) && $channel) {
+
 		// "New Item View" - show all items unthreaded in reverse created date order
 
 		$items = q("SELECT item.*, item.id AS item_id FROM item
-- 
cgit v1.2.3