revise content import/export part 2

6 files changed, 42 insertions, 97 deletions

diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php
index 72b8e34ce..ff6956051 100644
--- a/Zotlabs/Daemon/Content_importer.php
+++ b/Zotlabs/Daemon/Content_importer.php
@@ -23,7 +23,7 @@ class Content_importer {
 
 		$channel = channelx_by_nick($channel_address);
 		if(! $channel) {
-			logger('itemhelper: channel not found');
+			logger('channel not found');
 			return;
 		}
 
@@ -44,7 +44,6 @@ class Content_importer {
 		}
 
 		$j = json_decode($x['body'],true);
-
 		if (! $j) {
 			return;
 		}
@@ -52,8 +51,19 @@ class Content_importer {
 		if(! is_array($j['item']) || ! count($j['item']))
 			return;
 
+		//$total_pages = floor(intval($j['items_total']) / intval($j['items_page']));
+		//logger('importing items: ' . floor((intval($page) * 100) / $total_pages) . '%');
+
+		$saved_notification_flags = notifications_off($channel['channel_id']);
+
 		import_items($channel,$j['item'],false,((array_key_exists('relocate',$j)) ? $j['relocate'] : null));
 
+		notifications_on($channel['channel_id'], $saved_notification_flags);
+
+		$page++;
+
+		Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]);
+
 		return;
 	}
 }
diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php
index 883759a39..2be946fc3 100644
--- a/Zotlabs/Daemon/File_importer.php
+++ b/Zotlabs/Daemon/File_importer.php
@@ -14,7 +14,7 @@ class File_importer {
 
 		cli_startup();
 
-		$attach_id = $argv[1];
+		$page = $argv[1];
 		$channel_address = $argv[2];
 		$hz_server = urldecode($argv[3]);
 
@@ -22,28 +22,36 @@ class File_importer {
 
 		$channel = channelx_by_nick($channel_address);
 		if(! $channel) {
-			logger('filehelper: channel not found');
+			logger('channel not found');
 			return;
 		}
 
 		$headers = [
 			'X-API-Token'      => random_string(),
-			'X-API-Request'    => $hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id,
+			'X-API-Request'    => $hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page,
 			'Host'             => $m['host'],
-			'(request-target)' => 'get /api/z/1.0/file/export?f=&file_id=' . $attach_id,
+			'(request-target)' => 'get /api/z/1.0/file/export_page?f=records=1&page=' . $page,
 		];
 
 		$headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512');
-		$x = z_fetch_url($hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id,false,$redirects,[ 'headers' => $headers ]);
+
+		$x = z_fetch_url($hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, false, $redirects, [ 'headers' => $headers ]);
 
 		if(! $x['success']) {
 			logger('no API response',LOGGER_DEBUG);
-			return;
+			killme();
 		}
 
 		$j = json_decode($x['body'],true);
 
-		$r = sync_files($channel,[$j]);
+		if(! is_array($j[0]['attach']) || ! count($j[0]['attach']))
+			return;
+
+		$r = sync_files($channel,$j);
+
+		$page++;
+
+		Master::Summon([ 'File_importer',sprintf('%d',$page), $channel['channel_address'], urlencode($hz_server) ]);
 
 		return;
 	}
diff --git a/Zotlabs/Module/Import.php b/Zotlabs/Module/Import.php
index ecbcc3034..a4697a426 100644
--- a/Zotlabs/Module/Import.php
+++ b/Zotlabs/Module/Import.php
@@ -509,8 +509,6 @@ class Import extends Controller {
 		$addon = array('channel' => $channel, 'data' => $data);
 		call_hooks('import_channel', $addon);
 
-		$saved_notification_flags = notifications_off($channel['channel_id']);
-
 		if ($import_posts && array_key_exists('item', $data) && $data['item'])
 			import_items($channel, $data['item'], false, $relocate);
 
@@ -526,98 +524,24 @@ class Import extends Controller {
 			$poll_interval = get_config('system','poll_interval',3);
 			$page = 0;
 
-			while (1) {
-				$headers = [
-					'X-API-Token'      => random_string(),
-					'X-API-Request'    => $hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page ,
-					'Host'             => $m['host'],
-					'(request-target)' => 'get /api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page ,
-				];
-
-				$headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512');
-
-				$x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]);
-
-				// logger('z_fetch: ' . print_r($x,true));
-
-				if (! $x['success']) {
-					logger('no API response');
-					break;
-				}
-
-				$j = json_decode($x['body'],true);
-
-				if (! $j) {
-					break;
-				}
-
-				if (! is_array($j['item']) || ! count($j['item'])) {
-					break;
-				}
-
-				Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]);
-				sleep($poll_interval);
-
-				$page ++;
-				continue;
-			}
-
-			$headers = [
-				'X-API-Token'      => random_string(),
-				'X-API-Request'    => $hz_server . '/api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until),
-				'Host'             => $m['host'],
-				'(request-target)' => 'get /api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until),
-			];
-
-			$headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512');
-
-			$x = z_fetch_url($hz_server . '/api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until),false,$redirects,[ 'headers' => $headers ]);
-
-			if (! $x['success']) {
-				logger('no API response');
-				return;
-			}
-
-			$j = json_decode($x['body'],true);
-
-			if (! $j) {
-				return;
-			}
-
-			if (! $j['success']) {
-				return;
-			}
-
-			$poll_interval = get_config('system','poll_interval',3);
-
-			if(count($j['results'])) {
-				$todo = count($j['results']);
-				logger('total to process: ' . $todo,LOGGER_DEBUG);
-
-				foreach($j['results'] as $jj) {
-					Master::Summon([ 'File_importer',$jj['hash'], $channel['channel_address'], urlencode($hz_server) ]);
-					sleep($poll_interval);
-				}
-			}
-
-			notice(t('Files and Posts imported.') . EOL);
+			Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]);
+			Master::Summon([ 'File_importer',sprintf('%d',$page), $channel['channel_address'], urlencode($hz_server) ]);
 
 		}
 
-		notifications_on($channel['channel_id'], $saved_notification_flags);
-
-		if (array_key_exists('item_id', $data) && $data['item_id'])
-			import_item_ids($channel, $data['item_id']);
+		// i do not think this is still used
+		//if (array_key_exists('item_id', $data) && $data['item_id'])
+		//	import_item_ids($channel, $data['item_id']);
 
 		// This will indirectly perform a refresh_all *and* update the directory
 
 		Master::Summon(array('Directory', $channel['channel_id']));
 
-
-		notice(t('Import completed.') . EOL);
-
 		change_channel($channel['channel_id']);
 
+		notice(t('Import of items and files in progress') . EOL);
+
+		// TODO: go away to a import progress page
 		goaway(z_root());
 	}
 
diff --git a/boot.php b/boot.php
index cdba24f25..2145b30b1 100644
--- a/boot.php
+++ b/boot.php
@@ -1441,9 +1441,12 @@ function os_mkdir($path, $mode = 0777, $recursive = false) {
  */
 function rrmdir($path) {
 	if(is_dir($path) === true) {
-		$files = array_diff(scandir($path), array('.', '..'));
-		foreach($files as $file) {
-			rrmdir(realpath($path) . '/' . $file);
+		$dir_entries = scandir($path);
+		if (is_array($dir_entries)) {
+			$files = array_diff($dir_entries, array('.', '..'));
+			foreach($files as $file) {
+				rrmdir(realpath($path) . '/' . $file);
+			}
 		}
 		return rmdir($path);
 	}
diff --git a/include/attach.php b/include/attach.php
index ef3d6e242..067da21ae 100644
--- a/include/attach.php
+++ b/include/attach.php
@@ -2442,7 +2442,6 @@ function attach_export_data($channel, $resource_id, $deleted = false, $zap_compa
 	return $ret;
 }
 
-
 /**
  * @brief Strip off 'store/nickname/' from the provided path
  *
diff --git a/include/import.php b/include/import.php
index ae7b2787a..f635f3705 100644
--- a/include/import.php
+++ b/include/import.php
@@ -1265,6 +1265,7 @@ function sync_files($channel, $files) {
 
 						$store_path = $newfname;
 
+
 						$fp = fopen($newfname,'w');
 						if(! $fp) {
 							logger('failed to open storage file.',LOGGER_NORMAL,LOG_ERR);