Merge branch 'dev' of https://framagit.org/hubzilla/core into dev

12 files changed, 295 insertions, 137 deletions

diff --git a/include/account.php b/include/account.php
index 991e799b5..98d7f00a8 100644
--- a/include/account.php
+++ b/include/account.php
@@ -34,18 +34,29 @@ function check_account_email($email) {
 	if(! strlen($email))
 		return $result;
 
-	if(! validate_email($email))
-		$result['message'] .= t('Email address not valid') . EOL;
-	elseif(! allowed_email($email))
-		$result['message'] = t('Your email domain is not among those allowed on this site');
+	if(! validate_email($email)) {
+		$result['message'] = t('The provided email address is not valid');
+	}
+	elseif(! allowed_email($email)) {
+		$result['message'] = t('The provided email domain is not among those allowed on this site');
+	}
 	else {
-		$r = q("select account_email from account where account_email = '%s' limit 1",
+		$account = q("select account_email from account where account_email = '%s' limit 1",
+			dbesc($email)
+		);
+		if ($account) {
+			$result['message'] = t('The provided email address is already registered at this site');
+		}
+
+		$register = q("select reg_did2 from register where reg_vital = 1 and reg_did2 = '%s' and reg_didx = 'e' limit 1",
 			dbesc($email)
 		);
-		if($r) {
-			$result['message'] .= t('Your email address is already registered at this site.');
+		if ($register) {
+			$result['message'] = t('There is a pending registration for this address - click "Register" to continue verification');
+			$result['email_unverified'] = true;
 		}
 	}
+
 	if($result['message'])
 		$result['error'] = true;
 
@@ -851,12 +862,12 @@ function verify_register_scheme() {
 
 		if ($dbc[0]=='id') {
 			// v1 format
-			q("START TRANSACTION");
+			dbq("START TRANSACTION");
 
 			if(ACTIVE_DBTYPE == DBTYPE_POSTGRES) {
-				$r1 = q("ALTER TABLE register RENAME TO register100;");
+				$r1 = dbq("ALTER TABLE register RENAME TO register100;");
 
-				$r2 = q("CREATE TABLE register ("
+				$r2 = dbq("CREATE TABLE register ("
 					. "reg_id      serial  NOT NULL,"
 					. "reg_vital   int     DEFAULT 1 NOT NULL,"
 					. "reg_flags   bigint  DEFAULT 0 NOT NULL,"
@@ -864,9 +875,9 @@ function verify_register_scheme() {
 					. "reg_did2    text    DEFAULT '' NOT NULL,"
 					. "reg_hash    text    DEFAULT '' NOT NULL,"
 					. "reg_email   text    DEFAULT '' NOT NULL,"
-					. "reg_created timestamp  NOT NULL,"
-					. "reg_startup timestamp  NOT NULL,"
-					. "reg_expires timestamp  NOT NULL,"
+					. "reg_created timestamp NOT NULL DEFAULT '0001-01-01 00:00:00',"
+					. "reg_startup timestamp NOT NULL DEFAULT '0001-01-01 00:00:00',"
+					. "reg_expires timestamp NOT NULL DEFAULT '0001-01-01 00:00:00',"
 					. "reg_byc     bigint  DEFAULT 0 NOT NULL,"
 					. "reg_uid     bigint  DEFAULT 0 NOT NULL,"
 					. "reg_atip    text    DEFAULT '' NOT NULL,"
@@ -875,29 +886,29 @@ function verify_register_scheme() {
 					. "reg_stuff   text    NOT NULL,"
 					. "PRIMARY KEY (reg_id) );"
 				);
-				$r0 = q("CREATE INDEX ix_reg_vital ON register (reg_vital);");
-				$r0 = q("CREATE INDEX ix_reg_flags ON register (reg_flags);");
-				$r0 = q("CREATE INDEX ix_reg_didx ON register (reg_didx);");
-				$r0 = q("CREATE INDEX ix_reg_did2 ON register (reg_did2);");
-				$r0 = q("CREATE INDEX ix_reg_hash ON register (reg_hash);");
-				$r0 = q("CREATE INDEX ix_reg_email ON register (reg_email);");
-				$r0 = q("CREATE INDEX ix_reg_created ON register (reg_created);");
-				$r0 = q("CREATE INDEX ix_reg_startup ON register (reg_startup);");
-				$r0 = q("CREATE INDEX ix_reg_expires ON register (reg_expires);");
-				$r0 = q("CREATE INDEX ix_reg_byc ON register (reg_byc);");
-				$r0 = q("CREATE INDEX ix_reg_uid ON register (reg_uid);");
-				$r0 = q("CREATE INDEX ix_reg_atip ON register (reg_atip);");
-
-				$r3 = q("INSERT INTO register (reg_id, reg_hash, reg_created, reg_uid, reg_pass, reg_lang, reg_stuff) "
+				$r0 = dbq("CREATE INDEX ix_reg_vital ON register (reg_vital);");
+				$r0 = dbq("CREATE INDEX ix_reg_flags ON register (reg_flags);");
+				$r0 = dbq("CREATE INDEX ix_reg_didx ON register (reg_didx);");
+				$r0 = dbq("CREATE INDEX ix_reg_did2 ON register (reg_did2);");
+				$r0 = dbq("CREATE INDEX ix_reg_hash ON register (reg_hash);");
+				$r0 = dbq("CREATE INDEX ix_reg_email ON register (reg_email);");
+				$r0 = dbq("CREATE INDEX ix_reg_created ON register (reg_created);");
+				$r0 = dbq("CREATE INDEX ix_reg_startup ON register (reg_startup);");
+				$r0 = dbq("CREATE INDEX ix_reg_expires ON register (reg_expires);");
+				$r0 = dbq("CREATE INDEX ix_reg_byc ON register (reg_byc);");
+				$r0 = dbq("CREATE INDEX ix_reg_uid ON register (reg_uid);");
+				$r0 = dbq("CREATE INDEX ix_reg_atip ON register (reg_atip);");
+
+				$r3 = dbq("INSERT INTO register (reg_id, reg_hash, reg_created, reg_uid, reg_pass, reg_lang, reg_stuff) "
 					. "SELECT id, hash, created, uid, password, lang, '' FROM register100;");
 
-				$r4 = q("DROP TABLE register100");
+				$r4 = dbq("DROP TABLE register100");
 
 			}
 			else {
-				$r1 = q("RENAME TABLE register TO register100;");
+				$r1 = dbq("RENAME TABLE register TO register100;");
 
-				$r2 = q("CREATE TABLE IF NOT EXISTS register ("
+				$r2 = dbq("CREATE TABLE IF NOT EXISTS register ("
 	 				. "reg_id 		int(10) UNSIGNED 	NOT NULL AUTO_INCREMENT,"
   					. "reg_vital	int(10) UNSIGNED 	NOT NULL DEFAULT 1,"
   					. "reg_flags	int(10) UNSIGNED 	NOT NULL DEFAULT 0,"
@@ -930,10 +941,10 @@ function verify_register_scheme() {
 					. ") ENGINE=InnoDB  DEFAULT CHARSET=utf8mb4;"
 				);
 
-				$r3 = q("INSERT INTO register (reg_id, reg_hash, reg_created, reg_uid, reg_pass, reg_lang, reg_stuff) "
+				$r3 = dbq("INSERT INTO register (reg_id, reg_hash, reg_created, reg_uid, reg_pass, reg_lang, reg_stuff) "
 					. "SELECT id, hash, created, uid, password, lang, '' FROM register100;");
 
-				$r4 = q("DROP TABLE register100");
+				$r4 = dbq("DROP TABLE register100");
 			}
 
 			// $r = ($r1 && $r2 && $r3 && $r4);
@@ -941,11 +952,11 @@ function verify_register_scheme() {
 			$r = $r2;
 
 			if($r) {
-				q("COMMIT");
+				dbq("COMMIT");
 				return UPDATE_SUCCESS;
 			}
 
-			q("ROLLBACK");
+			dbq("ROLLBACK");
 			return UPDATE_FAILED;
 		}
 		elseif ( count($dbc) != 16 ) {
@@ -1195,8 +1206,15 @@ function get_account_techlevel($account_id = 0) {
 }
 
 function zar_log($msg='') {
-	file_put_contents('./zar.log',
-		 date('Y-m-d_H:i:s') . ' ' . $msg . ', ip: § ' . $_SERVER['REMOTE_ADDR'] . ' §' . "\n", FILE_APPEND);
+
+	if(get_config('system', 'register_logfile', 0)) {
+		file_put_contents('./zar.log',
+			date('Y-m-d_H:i:s') . ' ' . $msg . ', ip: § ' . $_SERVER['REMOTE_ADDR'] . ' §' . "\n", FILE_APPEND);
+	}
+	else {
+		logger('zar_log: ' . $msg . ', ip: § ' . $_SERVER['REMOTE_ADDR'] . ' §');
+	}
+
 	return;
 }
 
@@ -1261,21 +1279,25 @@ function zar_register_dutystate( $now=NULL, $day=NULL ) {
 
 }
 
-function get_pending_accounts() {
+function get_pending_accounts($get_all = false) {
 
-	/* get pending */
-	// [hilmar ->
-	//~ $r = q("SELECT account.*, reg_hash FROM account LEFT JOIN register ON account_id = reg_uid WHERE reg_vital = 1 AND (account_flags & %d) > 0",
-		//~ intval(ACCOUNT_PENDING)
-	//~ );
+	$sql_extra = " AND (reg_flags & " . ACCOUNT_UNVERIFIED . ") = 0 ";
 
-	// better useability at the moment to tell all (ACCOUNT_PENDING >= 0) instead of (> 0 for those need approval)
+	if($get_all)
+		$sql_extra = '';
 
-	$r = q("SELECT reg_did2, reg_created, reg_startup, reg_expires, reg_email, reg_atip, reg_hash, reg_id, reg_stuff
-		FROM register WHERE reg_vital = 1 AND (reg_flags & %d) = 0 AND (reg_flags & %d) >= 0",
-		intval(ACCOUNT_UNVERIFIED),
+	$r = q("SELECT reg_did2, reg_created, reg_startup, reg_expires, reg_email, reg_atip, reg_hash, reg_id, reg_flags, reg_stuff
+		FROM register WHERE reg_vital = 1 $sql_extra AND (reg_flags & %d) >= 0",
 		intval(ACCOUNT_PENDING)
 	);
 
 	return $r;
 }
+
+function remove_expired_registrations() {
+	q("DELETE FROM register WHERE (reg_expires < '%s' OR reg_expires = '%s') AND (reg_flags & %d) > 0",
+		dbesc(datetime_convert()),
+		dbesc(NULL_DATE),
+		dbesc(ACCOUNT_UNVERIFIED)
+	);
+}
diff --git a/include/channel.php b/include/channel.php
index cb6da9f13..85dd8ba02 100644
--- a/include/channel.php
+++ b/include/channel.php
@@ -922,7 +922,7 @@ function identity_basic_export($channel_id, $sections = null, $zap_compat = fals
 			$ret['photo'] = [
 				'type' => $r[0]['mimetype'],
 				'data' => (($r[0]['os_storage'])
-					? base64url_encode(file_get_contents($r[0]['content'])) : base64url_encode(dbunescbin($r[0]['content'])))
+					? base64url_encode(file_get_contents(dbunescbin($r[0]['content']))) : base64url_encode(dbunescbin($r[0]['content'])))
 			];
 		}
 	}
@@ -991,11 +991,11 @@ function identity_basic_export($channel_id, $sections = null, $zap_compat = fals
 		}
 
 		if($xchans) {
-			$r = q("select * from xchan where xchan_hash in ( " . implode(',',$xchans) . " ) ");
+			$r = dbq("select * from xchan where xchan_hash in ( " . implode(',',$xchans) . " ) ");
 			if($r)
 				$ret['xchan'] = $r;
 
-			$r = q("select * from hubloc where hubloc_hash in ( " . implode(',',$xchans) . " ) ");
+			$r = dbq("select * from hubloc where hubloc_hash in ( " . implode(',',$xchans) . " ) ");
 			if($r)
 				$ret['hubloc'] = $r;
 		}
@@ -1516,8 +1516,11 @@ function profile_load($nickname, $profile = '') {
 
 	if($p[0]['keywords']) {
 		$keywords = str_replace(array('#',',',' ',',,'),array('',' ',',',','),$p[0]['keywords']);
-		if(strlen($keywords) && $can_view_profile)
+		if(strlen($keywords) && $can_view_profile) {
+			if(! isset(App::$page['htmlhead']))
+				App::$page['htmlhead'] = '';
 			App::$page['htmlhead'] .= '<meta name="keywords" content="' . htmlentities($keywords,ENT_COMPAT,'UTF-8') . '" />' . "\r\n" ;
+		}
 	}
 
 	App::$profile = $p[0];
@@ -2593,7 +2596,7 @@ function channelx_by_n($id) {
 	}
 
 	$r = q("SELECT * FROM channel LEFT JOIN xchan ON channel_hash = xchan_hash WHERE channel_id = %d AND channel_removed = 0 LIMIT 1",
-		dbesc($id)
+		intval($id)
 	);
 
 	return(($r) ? $r[0] : false);
diff --git a/include/event.php b/include/event.php
index 765086167..440f559da 100644
--- a/include/event.php
+++ b/include/event.php
@@ -862,15 +862,15 @@ function event_import_ical($ical, $uid) {
 		// we do not have it here since parse_ical_file() is passing the vevent only.
 		$timezone_obj = \Sabre\VObject\TimeZoneUtil::getTimeZone($ical->DTSTART['TZID']);
 		$timezone = $timezone_obj->getName();
-		$ev['timezone'] = $timezone;
+		$ev['timezone'] = (($timezone) ? $timezone : date_default_timezone_get());
 	}
 
-	$ev['dtstart'] = datetime_convert((($ev['adjust']) ? 'UTC' : date_default_timezone_get()),$ev['timezone'],
+	$ev['dtstart'] = datetime_convert((($ev['adjust']) ? 'UTC' : $ev['timezone']), 'UTC',
 		$dtstart->format(\DateTime::W3C));
 
 	if(isset($ical->DTEND)) {
 		$dtend = $ical->DTEND->getDateTime();
-		$ev['dtend'] = datetime_convert((($ev['adjust']) ? 'UTC' : date_default_timezone_get()),$ev['timezone'],
+		$ev['dtend'] = datetime_convert((($ev['adjust']) ? 'UTC' : $ev['timezone']), 'UTC',
 			$dtend->format(\DateTime::W3C));
 	}
 	else {
@@ -1311,9 +1311,9 @@ function event_store_item($arr, $event) {
 		}
 
 		// propagate the event resource_id so that posts containing it are easily searchable in downstream copies
-		// of the item which have not stored the actual event. Required for Diaspora event federation as Diaspora 
+		// of the item which have not stored the actual event. Required for Diaspora event federation as Diaspora
 		// event_participation messages refer to the event resource_id as a parent, while out own event attendance
-		// activities refer to the item message_id as the parent. 
+		// activities refer to the item message_id as the parent.
 
 		set_iconfig($item_arr, 'system','event_id',$event['event_hash'],true);
 
diff --git a/include/feedutils.php b/include/feedutils.php
index 6cae14a01..f489030b6 100644
--- a/include/feedutils.php
+++ b/include/feedutils.php
@@ -296,7 +296,7 @@ function get_atom_author($feed, $item) {
 
 	$rawauthor = $item->get_item_tags(SIMPLEPIE_NAMESPACE_ATOM_10,'author');
 
-	if($rawauthor && $rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link']) {
+	if($rawauthor && isset($rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link'])) {
 		$base = $rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link'];
 		foreach($base as $link) {
 			if(!x($author, 'author_photo') || ! $author['author_photo']) {
@@ -324,7 +324,7 @@ function get_atom_author($feed, $item) {
 
 	// check for a yahoo media element (github etc.)
 
-	if(! $author['author_photo']) {
+	if(! x($author,'author_photo') || ! $author['author_photo']) {
 		$rawmedia = $item->get_item_tags(NAMESPACE_YMEDIA,'thumbnail');
 		if($rawmedia && $rawmedia[0]['attribs']['']['url']) {
 			$author['author_photo'] = strip_tags(unxmlify($rawmedia[0]['attribs']['']['url']));
@@ -334,9 +334,9 @@ function get_atom_author($feed, $item) {
 
 	// No photo/profile-link on the item - look at the feed level
 
-	if((! (x($author,'author_link'))) || (! (x($author,'author_photo')))) {
+	if(! x($author,'author_link') || ! x($author,'author_photo')) {
 		$rawauthor = $feed->get_feed_tags(SIMPLEPIE_NAMESPACE_ATOM_10,'author');
-		if($rawauthor && $rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link']) {
+		if($rawauthor && isset($rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link'])) {
 			$base = $rawauthor[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link'];
 			foreach($base as $link) {
 				if($link['attribs']['']['rel'] === 'alternate' && (! $author['author_link'])) {
@@ -372,16 +372,16 @@ function get_atom_author($feed, $item) {
 	if(! $rawowner)
 		$rawowner = $item->get_item_tags(NAMESPACE_ZOT, 'owner');
 
-	if($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['name'][0]['data'])
+	if(isset($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['name'][0]['data']))
 		$author['owner_name'] = unxmlify($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['name'][0]['data']);
-	elseif($rawowner[0]['child'][NAMESPACE_DFRN]['name'][0]['data'])
+	elseif(isset($rawowner[0]['child'][NAMESPACE_DFRN]['name'][0]['data']))
 		$author['owner_name'] = unxmlify($rawowner[0]['child'][NAMESPACE_DFRN]['name'][0]['data']);
-	if($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['uri'][0]['data'])
+	if(isset($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['uri'][0]['data']))
 		$author['owner_link'] = unxmlify($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['uri'][0]['data']);
-	elseif($rawowner[0]['child'][NAMESPACE_DFRN]['uri'][0]['data'])
+	elseif(isset($rawowner[0]['child'][NAMESPACE_DFRN]['uri'][0]['data']))
 		$author['owner_link'] = unxmlify($rawowner[0]['child'][NAMESPACE_DFRN]['uri'][0]['data']);
 
-	if($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link']) {
+	if(isset($rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link'])) {
 		$base = $rawowner[0]['child'][SIMPLEPIE_NAMESPACE_ATOM_10]['link'];
 
 		foreach($base as $link) {
@@ -478,13 +478,13 @@ function get_atom_elements($feed, $item) {
 	if($rawedited)
 		$res['edited'] = unxmlify($rawedited[0]['data']);
 
-	if((x($res,'edited')) && (! (x($res,'created'))))
+	if(x($res,'edited') && ! x($res,'created'))
 		$res['created'] = $res['edited'];
 
-	if(! $res['created'])
+	if(! x($res,'created'))
 		$res['created'] = $item->get_date('c');
 
-	if(! $res['edited'])
+	if(! x($res,'edited'))
 		$res['edited'] = $item->get_date('c');
 
 	$rawverb = $item->get_item_tags(NAMESPACE_ACTIVITY, 'verb');
@@ -509,7 +509,7 @@ function get_atom_elements($feed, $item) {
 		}
 	}
 
-	$ostatus_protocol = (($ostatus_conversation || $res['verb']) ? true : false);
+	$ostatus_protocol = ($ostatus_conversation || (x($res,'verb') && $res['verb']) ? true : false);
 
 	$mastodon = (($item->get_item_tags('http://mastodon.social/schema/1.0','scope')) ? true : false);
 	if($mastodon) {
@@ -1113,7 +1113,8 @@ function consume_feed($xml, $importer, &$contact, $pass = 0) {
 				$parent_link = $rawthread[0]['attribs']['']['href'];
 			}
 
-			logger('in-reply-to: ' . $parent_mid, LOGGER_DEBUG);
+			if(isset($parent_mid))
+				logger('in-reply-to: ' . $parent_mid, LOGGER_DEBUG);
 
 			if($is_reply) {
 
@@ -1408,7 +1409,7 @@ function consume_feed($xml, $importer, &$contact, $pass = 0) {
 
 				// if we have everything but a photo, provide the default profile photo
 
-				if($author['author_name'] && $author['author_link'] && (! $author['author_photo']))
+				if($author['author_name'] && $author['author_link'] && (! x($author,'author_photo') || ! $author['author_photo']))
 					$author['author_photo'] = z_root() . '/' . get_default_profile_photo(80);
 
 				if(is_array($contact)) {
diff --git a/include/hubloc.php b/include/hubloc.php
index e803b7852..2cce7a725 100644
--- a/include/hubloc.php
+++ b/include/hubloc.php
@@ -172,6 +172,46 @@ function remove_obsolete_hublocs() {
 	}
 }
 
+/**
+ * @brief Remove duplicate singleton hublocs
+ *
+ * This should not actually happen but it appears it does - probably due to race conditions.
+ * This function will just leave the hubloc with the highest id (latest)
+ *
+ * TODO: we should probably do something about that at the DB level.
+ *
+ */
+function remove_duplicate_singleton_hublocs() {
+	$hublocs = dbq("SELECT hubloc_hash, COUNT(hubloc_hash) FROM hubloc WHERE
+		hubloc_network IN(
+			'activitypub',
+			'diaspora',
+			'friendica-over-diaspora',
+			'gnusoc'
+		)
+		GROUP BY hubloc_hash
+		HAVING COUNT(hubloc_hash) > 1"
+	);
+
+	foreach($hublocs as $hubloc) {
+		$hubloc_hash = $hubloc['hubloc_hash'];
+
+		$max_id = q("select max(hubloc_id) as max_id from hubloc where hubloc_hash = '%s'",
+			dbesc($hubloc_hash)
+		);
+
+		$id = $max_id[0]['max_id'];
+
+		if($hubloc_hash && $id) {
+			q("delete from hubloc where hubloc_hash = '%s' and hubloc_id != %d",
+				dbesc($hubloc_hash),
+				intval($id)
+			);
+		}
+	}
+
+}
+
 
 /**
  * @brief Change primary hubloc.
@@ -191,7 +231,7 @@ function hubloc_change_primary($hubloc) {
 
 	logger('setting primary: ' . $hubloc['hubloc_url'] . ((intval($hubloc['hubloc_primary'])) ? '  true' : ' false'));
 
-	// See if this is a local hubloc and if so update the primary for the corresponding channel record. 
+	// See if this is a local hubloc and if so update the primary for the corresponding channel record.
 
 	if($hubloc['hubloc_url'] === z_root()) {
 		$r = q("select channel_id from channel where channel_hash = '%s' limit 1",
@@ -205,7 +245,7 @@ function hubloc_change_primary($hubloc) {
 		}
 	}
 
-	// we only need to proceed further if this particular hubloc is now primary 
+	// we only need to proceed further if this particular hubloc is now primary
 
 	if(! (intval($hubloc['hubloc_primary']))) {
 		logger('not primary: ' . $hubloc['hubloc_url']);
@@ -273,7 +313,7 @@ function hubloc_mark_as_down($posturl) {
  *
  * @param string $netid network identity (typically xchan_hash or hubloc_hash)
  * @return string
- */ 
+ */
 
 function locations_by_netid($netid) {
 
@@ -281,7 +321,7 @@ function locations_by_netid($netid) {
 		dbesc($netid)
 	);
 
-	
+
 	return array_elm_to_str($locs,'location',', ','trim_and_unpunify');
 
 }
@@ -331,7 +371,7 @@ function z6_discover() {
 				if ($q2) {
 					continue;
 				}
-				// zot6 hubloc not found. 
+				// zot6 hubloc not found.
 				if(strpos($q['site_project'],'hubzilla') !== false && version_compare($q['site_version'],'4.0') >= 0) {
 					// probe and store results - only for zot6 (over-ride the zot default)
 					discover_by_webbie($q['hubloc_addr'],'zot6');
diff --git a/include/import.php b/include/import.php
index 8ce582ede..379789109 100644
--- a/include/import.php
+++ b/include/import.php
@@ -708,25 +708,21 @@ function import_items($channel, $items, $sync = false, $relocate = null) {
 		$allow_code = channel_codeallowed($channel['channel_id']);
 
 		$deliver = false; // Don't deliver any messages or notifications when importing
-
 		foreach($items as $i) {
 			$item_result = false;
 			$item = get_item_elements($i,$allow_code);
-			if(! $item)
+			if(! $item) {
 				continue;
-
-			// deprecated
-
-			if(array_key_exists('diaspora_meta',$item))
-				unset($item['diaspora_meta']);
+			}
 
 			if($relocate && $item['mid'] === $item['parent_mid']) {
 				item_url_replace($channel,$item,$relocate['url'],z_root(),$relocate['channel_address']);
 			}
 
-			$r = q("select id, edited from item where mid = '%s' and uid = %d limit 1",
+			$r = q("select id, edited from item where mid = '%s' and uid = %d and revision = %d limit 1",
 				dbesc($item['mid']),
-				intval($channel['channel_id'])
+				intval($channel['channel_id']),
+				intval($item['revision'])
 			);
 			if($r) {
 
@@ -734,7 +730,7 @@ function import_items($channel, $items, $sync = false, $relocate = null) {
 				// so force an update even if we have the same timestamp
 
 				if($item['edited'] >= $r[0]['edited']) {
-					$item['id'] = $r[0]['id'];
+					$item['id']  = $r[0]['id'];
 					$item['uid'] = $channel['channel_id'];
 					$item_result = item_store_update($item,$allow_code,$deliver);
 				}
diff --git a/include/items.php b/include/items.php
index 8b22a7b4c..d6dd517ba 100644
--- a/include/items.php
+++ b/include/items.php
@@ -702,7 +702,6 @@ function get_item_elements($x,$allow_code = false) {
 	$arr['mid']          = (($x['message_id'])     ? htmlspecialchars($x['message_id'],     ENT_COMPAT,'UTF-8',false) : '');
 	$arr['parent_mid']   = (($x['message_top'])    ? htmlspecialchars($x['message_top'],    ENT_COMPAT,'UTF-8',false) : '');
 	$arr['thr_parent']   = (($x['message_parent']) ? htmlspecialchars($x['message_parent'], ENT_COMPAT,'UTF-8',false) : '');
-
 	$arr['plink']        = (($x['permalink'])      ? htmlspecialchars($x['permalink'],      ENT_COMPAT,'UTF-8',false) : '');
 	$arr['location']     = (($x['location'])       ? htmlspecialchars($x['location'],       ENT_COMPAT,'UTF-8',false) : '');
 	$arr['coord']        = (($x['longlat'])        ? htmlspecialchars($x['longlat'],        ENT_COMPAT,'UTF-8',false) : '');
@@ -718,17 +717,12 @@ function get_item_elements($x,$allow_code = false) {
 	$arr['comment_policy'] = (($x['comment_scope']) ? htmlspecialchars($x['comment_scope'], ENT_COMPAT,'UTF-8',false) : 'contacts');
 
 	$arr['sig']          = (($x['signature']) ? htmlspecialchars($x['signature'],  ENT_COMPAT,'UTF-8',false) : '');
-
 	$arr['obj']          = activity_sanitise($x['object']);
 	$arr['target']       = activity_sanitise($x['target']);
-
 	$arr['attach']       = activity_sanitise($x['attach']);
 	$arr['term']         = decode_tags($x['tags']);
 	$arr['iconfig']      = decode_item_meta($x['meta']);
-
-	$arr['item_private'] = ((array_key_exists('flags',$x) && is_array($x['flags']) && in_array('private',$x['flags'])) ? 1 : 0);
-
-	$arr['item_flags'] = 0;
+	$arr['item_flags']   = 0;
 
 	if(array_key_exists('flags',$x)) {
 
@@ -749,6 +743,12 @@ function get_item_elements($x,$allow_code = false) {
 		if(in_array('hidden',$x['flags']))
 			$arr['item_hidden'] = 1;
 
+		if(in_array('private', $x['flags']))
+			$arr['item_private'] = 1;
+
+		if(in_array('private', $x['flags']) && in_array('direct', $x['flags']))
+			$arr['item_private'] = 2;
+
 	}
 
 	// Here's the deal - the site might be down or whatever but if there's a new person you've never
@@ -758,22 +758,34 @@ function get_item_elements($x,$allow_code = false) {
 	// and not enough info to be able to look you up from your hash - which is the only thing stored with the post.
 
 	$xchan_hash = import_author_xchan($x['author']);
-	if($xchan_hash)
+	if($xchan_hash) {
 		$arr['author_xchan'] = $xchan_hash;
-	else
-		return array();
+	}
+	else {
+		return [];
+	}
 
 	// save a potentially expensive lookup if author == owner
+	$legacy_sig = false;
+	$owner_hash = '';
+	if(isset($x['owner']['id']) && isset($x['owner']['key']) && isset($x['owner']['network']) && $x['owner']['network'] === 'zot6') {
+		$owner_hash = Libzot::make_xchan_hash($x['owner']['id'], $x['owner']['key']);
+	}
+	else {
+		$owner_hash = make_xchan_hash($x['owner']['guid'],$x['owner']['guid_sig']);
+		$legacy_sig = true;
+	}
 
-	if($arr['author_xchan'] === make_xchan_hash($x['owner']['guid'],$x['owner']['guid_sig']))
+	if($arr['author_xchan'] === $owner_hash) {
 		$arr['owner_xchan'] = $arr['author_xchan'];
+	}
 	else {
 		$xchan_hash = import_author_xchan($x['owner']);
 		if($xchan_hash) {
 			$arr['owner_xchan'] = $xchan_hash;
 		}
 		else {
-			return array();
+			return [];
 		}
 	}
 
@@ -793,7 +805,15 @@ function get_item_elements($x,$allow_code = false) {
 		);
 		if($r) {
 			if($r[0]['xchan_pubkey'] && $r[0]['xchan_network'] === 'zot6') {
-				if(Libzot::verify($x['body'], $arr['sig'], $r[0]['xchan_pubkey'])) {
+				$item_verified = false;
+				if($legacy_sig) {
+					$item_verified = Crypto::verify($x['body'], base64url_decode($arr['sig']), $r[0]['xchan_pubkey']);
+				}
+				else {
+					$item_verified = Libzot::verify($x['body'], $arr['sig'], $r[0]['xchan_pubkey']);
+				}
+
+				if($item_verified) {
 					$arr['item_verified'] = 1;
 				}
 				else {
@@ -926,35 +946,54 @@ function import_author_xchan($x) {
 	 *   * \e string \b xchan_hash - Thre returned value
 	 */
 	call_hooks('import_author_xchan', $arr);
-	if($arr['xchan_hash'])
+	if($arr['xchan_hash']) {
 		return $arr['xchan_hash'];
+	}
 
 	$y = false;
 
-	if((! array_key_exists('network', $x)) || ($x['network'] === 'zot')) {
+	if((isset($x['id']) && isset($x['key'])) && (!isset($x['network']) || $x['network'] === 'zot6')) {
+		$y = Libzot::import_author_zot($x);
+	}
+
+	if(!$y && isset($x['url']) && isset($x['network']) && $x['network'] === 'zot6') {
+		$r = q("SELECT xchan_hash FROM xchan WHERE xchan_url = '%s' AND xchan_network = 'zot6'",
+			dbesc($x['url'])
+		);
+		if($r)
+			$y = $r[0]['xchan_hash'];
+		else
+			$y = discover_by_webbie($x['url'], 'zot6');
+	}
+
+	// if we were told that it's a zot6 connection, don't probe/import anything else
+
+	if($y)
+		return $y;
+
+	if(!isset($x['network']) || $x['network'] === 'zot') {
 		$y = import_author_zot($x);
 	}
 
-	// if we were told that it's a zot connection, don't probe/import anything else
-	if(array_key_exists('network',$x) && $x['network'] === 'zot') {
+	if(isset($x['network']) || $x['network'] === 'zot') {
 		if($x['url']) {
 			// check if we already have the zot6 xchan of this xchan_url. if not import it.
 			$r = q("SELECT xchan_hash FROM xchan WHERE xchan_url = '%s' AND xchan_network = 'zot6'",
 				dbesc($x['url'])
 			);
-
-			if(! $r)
+			// TODO: fix dupplicate with line 960
+			if(!$r)
 				discover_by_webbie($x['url'], 'zot6');
 		}
 
-		return $y;
+		if($y)
+			return $y;
+
 	}
 
 	// perform zot6 discovery
-
 	if($x['url']) {
-		$y = discover_by_webbie($x['url'],'zot6');
-
+		$y = discover_by_webbie($x['url'], 'zot6');
  		if($y) {
 			return $y;
 		}
@@ -968,7 +1007,7 @@ function import_author_xchan($x) {
 		$y = import_author_unknown($x);
 	}
 
-	return($y);
+	return $y;
 }
 
 /**
@@ -1296,7 +1335,17 @@ function encode_item_xchan($xchan) {
 	$ret['guid']     = $xchan['xchan_guid'];
 	$ret['guid_sig'] = $xchan['xchan_guid_sig'];
 
-	return $ret;
+	$ret['id']       = $xchan['xchan_guid'];
+	$ret['id_sig']   = $xchan['xchan_guid_sig'];
+	$ret['key']      = $xchan['xchan_pubkey'];
+
+	$hookdata = [
+		'encoded_xchan' => $ret
+	];
+
+	call_hooks('encode_item_xchan', $hookdata);
+
+	return $hookdata['encoded_xchan'];
 }
 
 function encode_item_terms($terms,$mirror = false) {
@@ -1511,6 +1560,8 @@ function encode_item_flags($item) {
 		$ret[] = 'obscured';
 	if(intval($item['item_private']))
 		$ret[] = 'private';
+	if(intval($item['item_private']) === 2)
+		$ret[] = 'direct';
 
 	return $ret;
 }
@@ -1690,14 +1741,14 @@ function item_sign(&$item) {
 	if(array_key_exists('sig',$item) && $item['sig'])
 		return;
 
-	$r = q("select channel_prvkey from channel where channel_id = %d and channel_hash = '%s' ",
+	$r = q("select * from channel where channel_id = %d and channel_hash = '%s' ",
 			intval($item['uid']),
 			dbesc($item['author_xchan'])
 	);
 	if(! $r)
 		return;
 
-	$item['sig'] = base64url_encode(Crypto::sign($item['body'], $r[0]['channel_prvkey']));
+	$item['sig'] = Libzot::sign($item['body'], $r[0]['channel_prvkey']);
 	$item['item_verified'] = 1;
 }
 
diff --git a/include/menu.php b/include/menu.php
index 88863f57b..1f65f987d 100644
--- a/include/menu.php
+++ b/include/menu.php
@@ -16,7 +16,7 @@ function menu_fetch($name,$uid,$observer_xchan) {
 	);
 	if($r) {
 		$x = q("select * from menu_item where mitem_menu_id = %d and mitem_channel_id = %d
-			$sql_options 
+			$sql_options
 			order by mitem_order asc, mitem_desc asc",
 			intval($r[0]['menu_id']),
 			intval($uid)
@@ -26,7 +26,7 @@ function menu_fetch($name,$uid,$observer_xchan) {
 
 	return null;
 }
-	
+
 function menu_element($channel,$menu) {
 
 	$arr = array();
@@ -67,7 +67,7 @@ function menu_element($channel,$menu) {
 			}
 			$arr['items'][] = $entry;
 		}
-	}	
+	}
 
 	return $arr;
 }
@@ -111,7 +111,7 @@ function menu_render($menu, $class='', $edit = false, $var = array()) {
 		$menu['items'][$x]['mitem_desc'] = zidify_links(smilies(bbcode($menu['items'][$x]['mitem_desc'])));
 	}
 
-	$wrap = (($var['wrap'] === 'none') ? false : true);
+	$wrap = ((isset($var['wrap']) && $var['wrap'] === 'none') ? false : true);
 
 	$ret = replace_macros(get_markup_template('usermenu.tpl'),array(
 		'$menu' => $menu['menu'],
@@ -168,7 +168,7 @@ function menu_create($arr) {
 
 	$t = datetime_convert();
 
-	$r = q("insert into menu ( menu_name, menu_desc, menu_flags, menu_channel_id, menu_created, menu_edited ) 
+	$r = q("insert into menu ( menu_name, menu_desc, menu_flags, menu_channel_id, menu_created, menu_edited )
 		values( '%s', '%s', %d, %d, '%s', '%s' )",
  		dbesc($menu_name),
 		dbesc($menu_desc),
@@ -260,7 +260,7 @@ function menu_edit($arr) {
 	}
 
 	return q("update menu set menu_name = '%s', menu_desc = '%s', menu_flags = %d, menu_edited = '%s'
-		where menu_id = %d and menu_channel_id = %d", 
+		where menu_id = %d and menu_channel_id = %d",
  		dbesc($menu_name),
 		dbesc($menu_desc),
 		intval($menu_flags),
@@ -295,7 +295,7 @@ function menu_delete_id($menu_id, $uid) {
 			intval($menu_id),
 			intval($uid)
 		);
-	}			
+	}
 	return false;
 }
 
@@ -304,11 +304,11 @@ function menu_add_item($menu_id, $uid, $arr) {
 
 	$mitem_link = escape_tags($arr['mitem_link']);
 	$mitem_desc = escape_tags($arr['mitem_desc']);
-	$mitem_order = intval($arr['mitem_order']);	
+	$mitem_order = intval($arr['mitem_order']);
 	$mitem_flags = intval($arr['mitem_flags']);
 
 	if(local_channel() == $uid) {
-		$channel = App::get_channel();	
+		$channel = App::get_channel();
 	}
 
 	$acl = new Zotlabs\Access\AccessList($channel);
@@ -344,12 +344,12 @@ function menu_edit_item($menu_id, $uid, $arr) {
 	$mitem_id = intval($arr['mitem_id']);
 	$mitem_link = escape_tags($arr['mitem_link']);
 	$mitem_desc = escape_tags($arr['mitem_desc']);
-	$mitem_order = intval($arr['mitem_order']);	
+	$mitem_order = intval($arr['mitem_order']);
 	$mitem_flags = intval($arr['mitem_flags']);
 
 
 	if(local_channel() == $uid) {
-		$channel = App::get_channel();	
+		$channel = App::get_channel();
 	}
 
 	$acl = new Zotlabs\Access\AccessList($channel);
@@ -403,7 +403,7 @@ function menu_sync_packet($uid,$observer_hash,$menu_id,$delete = false) {
 	$r = menu_fetch_id($menu_id,$uid);
 	$c = channelx_by_n($uid);
 	if($r) {
-		$m = menu_fetch($r['menu_name'],$uid,$observer_hash);	
+		$m = menu_fetch($r['menu_name'],$uid,$observer_hash);
 		if($m) {
 			if($delete)
 				$m['menu_delete'] = 1;
diff --git a/include/network.php b/include/network.php
index fcc7b4289..77b7c49d6 100644
--- a/include/network.php
+++ b/include/network.php
@@ -1118,9 +1118,9 @@ function discover_by_url($url, $arr = null) {
 /**
  * @brief
  *
- * @param string $webbie
+ * @param string $webbie (TODO: explain)
  * @param string $protocol (optional) default empty
- * @return boolean
+ * @return boolean|string (TODO: explain)
  */
 function discover_by_webbie($webbie, $protocol = '') {
 
diff --git a/include/opengraph.php b/include/opengraph.php
index 4c88570d3..465acbd39 100644
--- a/include/opengraph.php
+++ b/include/opengraph.php
@@ -62,6 +62,8 @@
                 $ogimagetype = $channel['xchan_photo_mimetype'];
         }
 
+	if (! isset(App::$page['htmlhead']))
+		App::$page['htmlhead'] = '';
         App::$page['htmlhead'] .= '<meta property="og:title" content="' . htmlspecialchars((isset($ogtitle) ? $ogtitle : $channel['channel_name'])) . '">' . "\r\n";
         App::$page['htmlhead'] .= '<meta property="og:image" content="' . $ogimage . '">' . "\r\n";
         App::$page['htmlhead'] .= '<meta property="og:image:type" content="' . $ogimagetype . '">' . "\r\n";
diff --git a/include/text.php b/include/text.php
index 749e1fd70..622c44f14 100644
--- a/include/text.php
+++ b/include/text.php
@@ -1108,7 +1108,7 @@ function magiclink_url($observer,$myaddr,$url) {
 
 function micropro($contact, $redirect = false, $class = '', $mode = false) {
 
-	if($contact['click'])
+	if(x($contact,'click'))
 		$url = '#';
 	else
 		$url = chanlink_hash($contact['xchan_hash']);
@@ -1121,10 +1121,10 @@ function micropro($contact, $redirect = false, $class = '', $mode = false) {
 		$tpl = 'micropro_card.tpl';
 
 	return replace_macros(get_markup_template($tpl), array(
-		'$click' => (($contact['click']) ? $contact['click'] : ''),
-		'$class' => $class . (($contact['archived']) ? ' archived' : ''),
-		'$oneway' => (($contact['oneway']) ? true : false),
-		'$perminfo' => $contact['perminfo'],
+		'$click' => (x($contact,'click') ? $contact['click'] : ''),
+		'$class' => $class . (x($contact,'archived') && $contact['archived'] ? ' archived' : ''),
+		'$oneway' => (x($contact,'oneway') && $contact['oneway'] ? true : false),
+		'$perminfo' => (x($contact,'perminfo') ? $contact['perminfo'] : ''),
 		'$url' => $url,
 		'$photo' => $contact['xchan_photo_s'],
 		'$name' => $contact['xchan_name'],
diff --git a/include/zid.php b/include/zid.php
index 75414a691..0a33280ee 100644
--- a/include/zid.php
+++ b/include/zid.php
@@ -89,20 +89,24 @@ function zid($s, $address = '') {
 }
 
 
-function strip_query_param($s,$param) {
-	return preg_replace('/[\?&]' . $param . '=(.*?)(&|$)/ism','$2',$s);
+function strip_query_param($s, $param) {
+	return drop_query_params($s, [$param]);
+	//return preg_replace('/[\?&]' . $param . '=(.*?)(&|$)/ism','$2',$s);
 }
 
 function strip_zids($s) {
-	return preg_replace('/[\?&]zid=(.*?)(&|$)/ism','$2',$s);
+	return drop_query_params($s, ['zid']);
+	//return preg_replace('/[\?&]zid=(.*?)(&|$)/ism','$2',$s);
 }
 
 function strip_owt($s) {
-	return preg_replace('/[\?&]owt=(.*?)(&|$)/ism','$2',$s);
+	return drop_query_params($s, ['owt']);
+	//return preg_replace('/[\?&]owt=(.*?)(&|$)/ism','$2',$s);
 }
 
 function strip_zats($s) {
-	return preg_replace('/[\?&]zat=(.*?)(&|$)/ism','$2',$s);
+	return drop_query_params($s, ['zat']);
+	//return preg_replace('/[\?&]zat=(.*?)(&|$)/ism','$2',$s);
 }
 
 function strip_escaped_zids($s) {
@@ -112,12 +116,51 @@ function strip_escaped_zids($s) {
 
 
 function clean_query_string($s = '') {
+
+	$x = (($s) ? $s : \App::$query_string);
+	return drop_query_params($x, ['zid', 'owt', 'zat', 'sort', 'f']);
+
+/*
 	$x = strip_zids(($s) ? $s : \App::$query_string);
 	$x = strip_owt($x);
 	$x = strip_zats($x);
 	$x = strip_query_param($x,'sort');
 
 	return strip_query_param($x,'f');
+*/
+}
+
+/**
+ * @brief Remove parameters from query string.
+ *
+ * @param string $s
+ *   The query string
+ * @param array $p
+ *   $p array of parameters to remove
+ * @return string
+ */
+
+function drop_query_params($s, $p) {
+		$parsed = parse_url($s);
+
+		$query = '';
+		$query_args = null;
+		if(isset($parsed['query'])) {
+			parse_str($parsed['query'], $query_args);
+		}
+
+		if(is_array($query_args)) {
+			foreach($query_args as $k => $v) {
+				if(in_array($k, $p))
+					continue;
+				$query .= (($query) ? '&' : '') . urlencode($k) . '=' . urlencode($v);
+			}
+		}
+
+		if($query)
+			$parsed['query'] = $query;
+
+		return unparse_url($parsed);
 }