channel export - use a selected list of functional data categories to export; this allows one to export single data sets instead of always exporting everything we know about

9 files changed, 280 insertions, 171 deletions

diff --git a/Zotlabs/Module/Import.php b/Zotlabs/Module/Import.php
index 52835c685..69e06d256 100644
--- a/Zotlabs/Module/Import.php
+++ b/Zotlabs/Module/Import.php
@@ -478,6 +478,9 @@ class Import extends \Zotlabs\Web\Controller {
 
 		if(is_array($data['wiki']))
 			import_items($channel,$data['wiki'],false,$relocate);
+
+		if(is_array($data['webpages']))
+			import_items($channel,$data['webpages'],false,$relocate);
 		
 		$addon = array('channel' => $channel,'data' => $data);
 		call_hooks('import_channel',$addon);
diff --git a/Zotlabs/Module/Uexport.php b/Zotlabs/Module/Uexport.php
index f36d77174..28c840ceb 100644
--- a/Zotlabs/Module/Uexport.php
+++ b/Zotlabs/Module/Uexport.php
@@ -9,10 +9,11 @@ class Uexport extends \Zotlabs\Web\Controller {
 			killme();
 	
 		if(argc() > 1) {
+
+			$sections = (($_REQUEST['sections']) ? explode(',',$_REQUEST['sections']) : '');
+
 			$channel = \App::get_channel();
-	
-			require_once('include/channel.php');
-	
+
 			if(argc() > 1 && intval(argv(1)) > 1900) {
 				$year = intval(argv(1));
 			}
@@ -30,15 +31,16 @@ class Uexport extends \Zotlabs\Web\Controller {
 			}
 	
 			if(argc() > 1 && argv(1) === 'basic') {
-				echo json_encode(identity_basic_export(local_channel()));
+				echo json_encode(identity_basic_export(local_channel(),$sections));
 				killme();
 			}
 	
-			// FIXME - this basically doesn't work in the wild with a channel more than a few months old due to memory and execution time limits.  
-			// It probably needs to be built at the CLI and offered to download as a tarball.  Maybe stored in the members dav.
+			// Warning: this option may consume a lot of memory
 	
 			if(argc() > 1 && argv(1) === 'complete') {
-				echo json_encode(identity_basic_export(local_channel(),true));
+				$sections = get_default_export_sections();
+				$sections[] = 'items';
+				echo json_encode(identity_basic_export(local_channel(),$sections));
 				killme();
 			}
 		}
@@ -57,6 +59,7 @@ class Uexport extends \Zotlabs\Web\Controller {
 			'$basic' => t('Export your basic channel information to a file.  This acts as a backup of your connections, permissions, profile and basic data, which can be used to import your data to a new server hub, but does not contain your content.'),
 			'$fulltitle' => t('Export Content'),
 			'$full' => t('Export your channel information and recent content to a JSON backup that can be restored or imported to another server hub. This backs up all of your connections, permissions, profile data and several months of posts. This file may be VERY large.  Please be patient - it may take several minutes for this download to begin.'),
+
 			'$by_year' => t('Export your posts from a given year.'),
 	
 			'$extra' => t('You may also export your posts and conversations for a particular year or month. Adjust the date in your browser location bar to select other dates. If the export fails (possibly due to memory exhaustion on your server hub), please try again selecting a more limited date range.'),
diff --git a/Zotlabs/Render/Comanche.php b/Zotlabs/Render/Comanche.php
index 048921670..5826063fd 100644
--- a/Zotlabs/Render/Comanche.php
+++ b/Zotlabs/Render/Comanche.php
@@ -99,6 +99,17 @@ class Comanche {
 		}
 	}
 
+	/**
+	 * Currently supported condition variables:
+	 *
+	 * $config.xxx.yyy - get_config with cat = xxx and k = yyy
+	 * $request - request uri for this page
+	 * $observer.language - viewer's preferred language (closest match)
+	 * $observer.address - xchan_addr or false
+	 * $observer.name - xchan_name or false
+	 * $observer - xchan_hash of observer or empty string
+	 */
+
 	function get_condition_var($v) {
 		if($v) {
 			$x = explode('.',$v);
diff --git a/doc/hook/get_default_export_sections b/doc/hook/get_default_export_sections
new file mode 100644
index 000000000..09b146643
--- /dev/null
+++ b/doc/hook/get_default_export_sections
@@ -0,0 +1,10 @@
+[h3]get_default_export_sections[/h3]
+
+The get_default_export_sections call returns the basic functional groups of data to export using channel_export_basic().
+
+The hook is passed an array 
+		[
+			'sections' => [ 'channel', 'connections', 'config', 'apps', 'chatrooms', 'events', 'webpages', 'mail', 'wikis' ]
+		]
+
+If you desire the export to contain three months of items, add 'items' to the 'sections' array
diff --git a/doc/hook/identity_basic_export.bb b/doc/hook/identity_basic_export.bb
index 3f4e88799..71329ba1e 100644
--- a/doc/hook/identity_basic_export.bb
+++ b/doc/hook/identity_basic_export.bb
@@ -1 +1,10 @@
 [h2]identity_basic_export[/h2]
+
+Called when exporting data for a channel
+Passed array contains
+
+	[ 
+		'channel_id' => channel_id being exported
+		'sections' => array of functional export sections which are being exported
+		'data' => the export data array which has been generated
+	]
diff --git a/doc/hooklist.bb b/doc/hooklist.bb
index c965b7c44..59d370117 100644
--- a/doc/hooklist.bb
+++ b/doc/hooklist.bb
@@ -242,6 +242,9 @@ Hooks allow plugins/addons to "hook into" the code at many points and alter the
 [zrl=[baseurl]/help/hook/get_best_language]get_best_language[/zrl]
 	called when choosing the preferred language for the page
 
+[zrl=[baseurl]/help/hook/get_default_export_sections]get_default_export_sections[/zrl]
+	Called to get the default list of functional data groups to export in identity_basic_export()
+
 [zrl=[baseurl]/help/hook/get_features]get_features[/zrl]
 	Called when get_features() is called
 
diff --git a/include/api_zot.php b/include/api_zot.php
index 0b10555a6..aaa9ee497 100644
--- a/include/api_zot.php
+++ b/include/api_zot.php
@@ -69,8 +69,13 @@
 			logger('api_export_basic: no user');
 			return false;
 		}
-		
-		json_return_and_die(identity_basic_export(api_user(),(($_REQUEST['posts']) ? intval($_REQUEST['posts']) : 0 )));	
+		$sections = (($_REQUEST['sections']) ? explode(',',$_REQUEST['sections']) : '');
+		if($_REQUEST['posts']) {
+			$sections = get_default_export_sections();
+			$sections[] = 'items';
+		}
+
+		json_return_and_die(identity_basic_export(api_user(),$sections));
 	}
 
 
diff --git a/include/channel.php b/include/channel.php
index 10c38a5ac..83cf97305 100644
--- a/include/channel.php
+++ b/include/channel.php
@@ -476,6 +476,16 @@ function set_default_login_identity($account_id, $channel_id, $force = true) {
 	}
 }
 
+
+function get_default_export_sections() {
+	$sections = [ 'channel', 'connections', 'config', 'apps', 'chatrooms', 'events', 'webpages', 'mail', 'wikis' ];
+
+	$cb = [ 'sections' => $sections ];
+	call_hooks('get_default_export_sections', $cb);
+	return $cb['sections'];
+}
+
+
 /**
  * @brief Create an array representing the important channel information
  * which would be necessary to create a nomadic identity clone. This includes
@@ -489,225 +499,279 @@ function set_default_login_identity($account_id, $channel_id, $force = true) {
  * @returns array
  *     See function for details
  */
-function identity_basic_export($channel_id, $items = false) {
+function identity_basic_export($channel_id, $sections = null) {
 
 	/*
-	 * Red basic channel export
+	 * basic channel export
 	 */
 
-	$ret = array();
+	if(! $sections) {
+		$sections = get_default_export_sections();
+	}
+		
+	$ret = [];
 
 	// use constants here as otherwise we will have no idea if we can import from a site
 	// with a non-standard platform and version.
-	$ret['compatibility'] = array('project' => PLATFORM_NAME, 'version' => STD_VERSION, 'database' => DB_UPDATE_VERSION, 'server_role' => Zotlabs\Lib\System::get_server_role());
+
+	$ret['compatibility'] = [
+		'project' => PLATFORM_NAME, 
+		'version' => STD_VERSION, 
+		'database' => DB_UPDATE_VERSION, 
+		'server_role' => Zotlabs\Lib\System::get_server_role()
+	];
+
+	/*
+	 * Process channel information regardless of it is one of the sections desired
+	 * because we need the channel relocation information in all export files/streams.
+	 */
 
 	$r = q("select * from channel where channel_id = %d limit 1",
 		intval($channel_id)
 	);
 	if($r) {
 		translate_channel_perms_outbound($r[0]);
-		$ret['channel'] = $r[0];
 		$ret['relocate'] = [ 'channel_address' => $r[0]['channel_address'], 'url' => z_root()];
-	}
-
-	$r = q("select * from profile where uid = %d",
-		intval($channel_id)
-	);
-	if($r)
-		$ret['profile'] = $r;
-
-	$xchans = array();
-	$r = q("select * from abook where abook_channel = %d ",
-		intval($channel_id)
-	);
-	if($r) {
-		$ret['abook'] = $r;
-
-		for($x = 0; $x < count($ret['abook']); $x ++) {
-			$xchans[] = $ret['abook'][$x]['abook_xchan'];
-			$abconfig = load_abconfig($channel_id,$ret['abook'][$x]['abook_xchan']);
-			if($abconfig)
-				$ret['abook'][$x]['abconfig'] = $abconfig;
-			translate_abook_perms_outbound($ret['abook'][$x]);
+		if(in_array('channel',$sections)) {
+			$ret['channel'] = $r[0];
 		}
-		stringify_array_elms($xchans);
 	}
 
-	if($xchans) {
-		$r = q("select * from xchan where xchan_hash in ( " . implode(',',$xchans) . " ) ");
+	if(in_array('channel',$sections)) {
+		$r = q("select * from profile where uid = %d",
+			intval($channel_id)
+		);
 		if($r)
-			$ret['xchan'] = $r;
+			$ret['profile'] = $r;
 
-		$r = q("select * from hubloc where hubloc_hash in ( " . implode(',',$xchans) . " ) ");
-		if($r)
-			$ret['hubloc'] = $r;
-	}
 
-	$r = q("select * from groups where uid = %d ",
-		intval($channel_id)
-	);
+		$r = q("select mimetype, content, os_storage from photo 
+			where imgscale = 4 and photo_usage = %d and uid = %d limit 1",
+			intval(PHOTO_PROFILE),
+			intval($channel_id)
+		);
 
-	if($r)
-		$ret['group'] = $r;
+		if($r) {
+			$ret['photo'] = [
+				'type' => $r[0]['mimetype'], 
+				'data' => (($r[0]['os_storage']) 
+					? base64url_encode(file_get_contents($r[0]['content'])) : base64url_encode($r[0]['content']))
+			];
+		}
+	}
 
-	$r = q("select * from group_member where uid = %d ",
-		intval($channel_id)
-	);
-	if($r)
-		$ret['group_member'] = $r;
+	if(in_array('connections',$sections)) {
+		$xchans = array();
+		$r = q("select * from abook where abook_channel = %d ",
+			intval($channel_id)
+		);
+		if($r) {
+			$ret['abook'] = $r;
+
+			for($x = 0; $x < count($ret['abook']); $x ++) {
+				$xchans[] = $ret['abook'][$x]['abook_xchan'];
+				$abconfig = load_abconfig($channel_id,$ret['abook'][$x]['abook_xchan']);
+				if($abconfig)
+					$ret['abook'][$x]['abconfig'] = $abconfig;
+				translate_abook_perms_outbound($ret['abook'][$x]);
+			}
+			stringify_array_elms($xchans);
+		}
 
-	$r = q("select * from pconfig where uid = %d",
-		intval($channel_id)
-	);
-	if($r)
-		$ret['config'] = $r;
+		if($xchans) {
+			$r = q("select * from xchan where xchan_hash in ( " . implode(',',$xchans) . " ) ");
+			if($r)
+				$ret['xchan'] = $r;
 
-	$r = q("select mimetype, content, os_storage from photo where imgscale = 4 and photo_usage = %d and uid = %d limit 1",
-		intval(PHOTO_PROFILE),
-		intval($channel_id)
-	);
+			$r = q("select * from hubloc where hubloc_hash in ( " . implode(',',$xchans) . " ) ");
+			if($r)
+				$ret['hubloc'] = $r;
+		}
 
-	if($r) {
-		$ret['photo'] = array('type' => $r[0]['mimetype'], 'data' => (($r[0]['os_storage']) ? base64url_encode(file_get_contents($r[0]['content'])) : base64url_encode($r[0]['content'])));
-	}
+		$r = q("select * from groups where uid = %d ",
+			intval($channel_id)
+		);
 
-	// All other term types will be included in items, if requested.
+		if($r)
+			$ret['group'] = $r;
 
-	$r = q("select * from term where ttype in (%d,%d) and uid = %d",
-		intval(TERM_SAVEDSEARCH),
-		intval(TERM_THING),
-		intval($channel_id)
-	);
-	if($r)
-		$ret['term'] = $r;
+		$r = q("select * from group_member where uid = %d ",
+			intval($channel_id)
+		);
+		if($r)
+			$ret['group_member'] = $r;
 
-	// add psuedo-column obj_baseurl to aid in relocations
+	}
 
-	$r = q("select obj.*, '%s' as obj_baseurl from obj where obj_channel = %d",
-		dbesc(z_root()),
-		intval($channel_id)
-	);
+	if(in_array('config',$sections)) {
+		$r = q("select * from pconfig where uid = %d",
+			intval($channel_id)
+		);
+		if($r)
+			$ret['config'] = $r;
+	
+		// All other term types will be included in items, if requested.
+
+		$r = q("select * from term where ttype in (%d,%d) and uid = %d",
+			intval(TERM_SAVEDSEARCH),
+			intval(TERM_THING),
+			intval($channel_id)
+		);
+		if($r)
+			$ret['term'] = $r;
 
-	if($r)
-		$ret['obj'] = $r;
+		// add psuedo-column obj_baseurl to aid in relocations
 
-	$r = q("select * from app where app_channel = %d and app_system = 0",
-		intval($channel_id)
-	);
-	if($r) {
-		for($x = 0; $x < count($r); $x ++) {
-			$r[$x]['term'] = q("select * from term where otype = %d and oid = %d",
-				intval(TERM_OBJ_APP),
-				intval($r[$x]['id'])
-			);
-		}
-		$ret['app'] = $r;
-	}
+		$r = q("select obj.*, '%s' as obj_baseurl from obj where obj_channel = %d",
+			dbesc(z_root()),
+			intval($channel_id)
+		);
 
-	$r = q("select * from chatroom where cr_uid = %d",
-		intval($channel_id)
-	);
-	if($r)
-		$ret['chatroom'] = $r;
+		if($r)
+			$ret['obj'] = $r;
 
-	$r = q("select * from event where uid = %d",
-		intval($channel_id)
-	);
-	if($r)
-		$ret['event'] = $r;
+		$r = q("select * from likes where channel_id = %d",
+			intval($channel_id)
+		);
+
+		if($r)
+			$ret['likes'] = $r;
 
-	$r = q("select * from item where resource_type = 'event' and uid = %d",
-		intval($channel_id)
-	);
-	if($r) {
-		$ret['event_item'] = array();
-		xchan_query($r);
-		$r = fetch_post_tags($r,true);
-		foreach($r as $rr)
-			$ret['event_item'][] = encode_item($rr,true);
 	}
 
-	$x = menu_list($channel_id);
-	if($x) {
-		$ret['menu'] = array();
-		for($y = 0; $y < count($x); $y ++) {
-			$m = menu_fetch($x[$y]['menu_name'],$channel_id,$ret['channel']['channel_hash']);
-			if($m)
-				$ret['menu'][] = menu_element($ret['channel'],$m);
+	if(in_array('apps',$sections)) {
+		$r = q("select * from app where app_channel = %d and app_system = 0",
+			intval($channel_id)
+		);
+		if($r) {
+			for($x = 0; $x < count($r); $x ++) {
+				$r[$x]['term'] = q("select * from term where otype = %d and oid = %d",
+					intval(TERM_OBJ_APP),
+					intval($r[$x]['id'])
+				);
+			}
+			$ret['app'] = $r;
 		}
 	}
 
-	$addon = array('channel_id' => $channel_id,'data' => $ret);
-	call_hooks('identity_basic_export',$addon);
-	$ret = $addon['data'];
+	if(in_array('chatrooms',$sections)) {
+		$r = q("select * from chatroom where cr_uid = %d",
+			intval($channel_id)
+		);
+		if($r)
+			$ret['chatroom'] = $r;
+	}
 
-	if(! $items)
-		return $ret;
 
-	$r = q("select * from likes where channel_id = %d",
-		intval($channel_id)
-	);
+	if(in_array('events',$sections)) {
+		$r = q("select * from event where uid = %d",
+			intval($channel_id)
+		);
+		if($r)
+			$ret['event'] = $r;
 
-	if($r)
-		$ret['likes'] = $r;
+		$r = q("select * from item where resource_type = 'event' and uid = %d",
+			intval($channel_id)
+		);
+		if($r) {
+			$ret['event_item'] = array();
+			xchan_query($r);
+			$r = fetch_post_tags($r,true);
+			foreach($r as $rr)
+				$ret['event_item'][] = encode_item($rr,true);
+		}
+	}
 
+	if(in_array('webpages',$sections)) {
+		$x = menu_list($channel_id);
+		if($x) {
+			$ret['menu'] = array();
+			for($y = 0; $y < count($x); $y ++) {
+				$m = menu_fetch($x[$y]['menu_name'],$channel_id,$ret['channel']['channel_hash']);
+				if($m)
+					$ret['menu'][] = menu_element($ret['channel'],$m);
+			}
+		}
+		$r = q("select * from item where item_type in ( " 
+			. ITEM_TYPE_BLOCK . "," . ITEM_TYPE_PDL . "," . ITEM_TYPE_WEBPAGE . " ) and uid = %d",
+			intval($channel_id)
+		);
+		if($r) {
+			$ret['webpages'] = array();
+			xchan_query($r);
+			$r = fetch_post_tags($r,true);
+			foreach($r as $rr)
+				$ret['webpages'][] = encode_item($rr,true);
 
-	$r = q("select * from conv where uid = %d",
-		intval($channel_id)
-	);
-	if($r) {
-		for($x = 0; $x < count($r); $x ++) {
-			$r[$x]['subject'] = base64url_decode(str_rot47($r[$x]['subject']));
 		}
-		$ret['conv'] = $r;
 	}
 
-	$r = q("select * from mail where mail.uid = %d",
-		intval($channel_id)
-	);
-	if($r) {
-		$m = array();
-		foreach($r as $rr) {
-			xchan_mail_query($rr);
-			$m[] = mail_encode($rr,true);
+	if(in_array('mail',$sections)) {
+		$r = q("select * from conv where uid = %d",
+			intval($channel_id)
+		);
+		if($r) {
+			for($x = 0; $x < count($r); $x ++) {
+				$r[$x]['subject'] = base64url_decode(str_rot47($r[$x]['subject']));
+			}
+			$ret['conv'] = $r;
 		}
-		$ret['mail'] = $m;
-	}
 
-	$r = q("select * from item where resource_type like 'nwiki%%' and uid = %d order by created",
-		intval($channel_id)
-	);
-	if($r) {
-		$ret['wiki'] = array();
-		xchan_query($r);
-		$r = fetch_post_tags($r,true);
-		foreach($r as $rv) {
-			$ret['wiki'][] = encode_item($rv,true);
+		$r = q("select * from mail where mail.uid = %d",
+			intval($channel_id)
+		);
+		if($r) {
+			$m = array();
+			foreach($r as $rr) {
+				xchan_mail_query($rr);
+				$m[] = mail_encode($rr,true);
+			}
+			$ret['mail'] = $m;
 		}
 	}
 
-	/** @warning this may run into memory limits on smaller systems */
+	if(in_array('wikis',$sections)) {
+		$r = q("select * from item where resource_type like 'nwiki%%' and uid = %d order by created",
+			intval($channel_id)
+		);
+		if($r) {
+			$ret['wiki'] = array();
+			xchan_query($r);
+			$r = fetch_post_tags($r,true);
+			foreach($r as $rv) {
+				$ret['wiki'][] = encode_item($rv,true);
+			}
+		}
+	}
 
+	if(in_array('items',$sections)) {
+		/** @warning this may run into memory limits on smaller systems */
 
-	/** export three months of posts. If you want to export and import all posts you have to start with
-	  * the first year and export/import them in ascending order.
-	  *
-	  * Don't export linked resource items. we'll have to pull those out separately.
-	  */
+		/** export three months of posts. If you want to export and import all posts you have to start with
+		 * the first year and export/import them in ascending order.
+		 *
+		 * Don't export linked resource items. we'll have to pull those out separately.
+		 */
 
-	$r = q("select * from item where item_wall = 1 and item_deleted = 0 and uid = %d and created > %s - INTERVAL %s and resource_type = '' order by created",
-		intval($channel_id),
-		db_utcnow(),
-		db_quoteinterval('3 MONTH')
-	);
-	if($r) {
-		$ret['item'] = array();
-		xchan_query($r);
-		$r = fetch_post_tags($r,true);
-		foreach($r as $rr)
-			$ret['item'][] = encode_item($rr,true);
+		$r = q("select * from item where item_wall = 1 and item_deleted = 0 and uid = %d 
+			and created > %s - INTERVAL %s and resource_type = '' order by created",
+			intval($channel_id),
+			db_utcnow(),
+			db_quoteinterval('3 MONTH')
+		);
+		if($r) {
+			$ret['item'] = array();
+			xchan_query($r);
+			$r = fetch_post_tags($r,true);
+			foreach($r as $rr)
+				$ret['item'][] = encode_item($rr,true);
+		}
 	}
 
+	$addon = [ 'channel_id' => $channel_id, 'sections' => $sections, 'data' => $ret];
+	call_hooks('identity_basic_export',$addon);
+	$ret = $addon['data'];
+
 	return $ret;
 }
 
diff --git a/include/text.php b/include/text.php
index 4f0dc869b..8c01ed1d2 100644
--- a/include/text.php
+++ b/include/text.php
@@ -3130,6 +3130,7 @@ function gen_link_id($mid) {
 	return $mid;
 }
 
+
 // callback for array_walk
 
 function array_trim(&$v,$k) {