From 4ba47698d7d0ef6c799bd0a28045c1d1e49c76f3 Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Wed, 29 Sep 2021 21:47:12 +0200 Subject: revise content import/export part 2 --- Zotlabs/Daemon/Content_importer.php | 14 +++++- Zotlabs/Daemon/File_importer.php | 22 ++++++--- Zotlabs/Module/Import.php | 92 ++++--------------------------------- boot.php | 9 ++-- include/attach.php | 1 - include/import.php | 1 + 6 files changed, 42 insertions(+), 97 deletions(-) diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php index 72b8e34ce..ff6956051 100644 --- a/Zotlabs/Daemon/Content_importer.php +++ b/Zotlabs/Daemon/Content_importer.php @@ -23,7 +23,7 @@ class Content_importer { $channel = channelx_by_nick($channel_address); if(! $channel) { - logger('itemhelper: channel not found'); + logger('channel not found'); return; } @@ -44,7 +44,6 @@ class Content_importer { } $j = json_decode($x['body'],true); - if (! $j) { return; } @@ -52,8 +51,19 @@ class Content_importer { if(! is_array($j['item']) || ! count($j['item'])) return; + //$total_pages = floor(intval($j['items_total']) / intval($j['items_page'])); + //logger('importing items: ' . floor((intval($page) * 100) / $total_pages) . '%'); + + $saved_notification_flags = notifications_off($channel['channel_id']); + import_items($channel,$j['item'],false,((array_key_exists('relocate',$j)) ? $j['relocate'] : null)); + notifications_on($channel['channel_id'], $saved_notification_flags); + + $page++; + + Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]); + return; } } diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php index 883759a39..2be946fc3 100644 --- a/Zotlabs/Daemon/File_importer.php +++ b/Zotlabs/Daemon/File_importer.php @@ -14,7 +14,7 @@ class File_importer { cli_startup(); - $attach_id = $argv[1]; + $page = $argv[1]; $channel_address = $argv[2]; $hz_server = urldecode($argv[3]); @@ -22,28 +22,36 @@ class File_importer { $channel = channelx_by_nick($channel_address); if(! $channel) { - logger('filehelper: channel not found'); + logger('channel not found'); return; } $headers = [ 'X-API-Token' => random_string(), - 'X-API-Request' => $hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id, + 'X-API-Request' => $hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, 'Host' => $m['host'], - '(request-target)' => 'get /api/z/1.0/file/export?f=&file_id=' . $attach_id, + '(request-target)' => 'get /api/z/1.0/file/export_page?f=records=1&page=' . $page, ]; $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512'); - $x = z_fetch_url($hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id,false,$redirects,[ 'headers' => $headers ]); + + $x = z_fetch_url($hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, false, $redirects, [ 'headers' => $headers ]); if(! $x['success']) { logger('no API response',LOGGER_DEBUG); - return; + killme(); } $j = json_decode($x['body'],true); - $r = sync_files($channel,[$j]); + if(! is_array($j[0]['attach']) || ! count($j[0]['attach'])) + return; + + $r = sync_files($channel,$j); + + $page++; + + Master::Summon([ 'File_importer',sprintf('%d',$page), $channel['channel_address'], urlencode($hz_server) ]); return; } diff --git a/Zotlabs/Module/Import.php b/Zotlabs/Module/Import.php index ecbcc3034..a4697a426 100644 --- a/Zotlabs/Module/Import.php +++ b/Zotlabs/Module/Import.php @@ -509,8 +509,6 @@ class Import extends Controller { $addon = array('channel' => $channel, 'data' => $data); call_hooks('import_channel', $addon); - $saved_notification_flags = notifications_off($channel['channel_id']); - if ($import_posts && array_key_exists('item', $data) && $data['item']) import_items($channel, $data['item'], false, $relocate); @@ -526,98 +524,24 @@ class Import extends Controller { $poll_interval = get_config('system','poll_interval',3); $page = 0; - while (1) { - $headers = [ - 'X-API-Token' => random_string(), - 'X-API-Request' => $hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , - 'Host' => $m['host'], - '(request-target)' => 'get /api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , - ]; - - $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512'); - - $x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]); - - // logger('z_fetch: ' . print_r($x,true)); - - if (! $x['success']) { - logger('no API response'); - break; - } - - $j = json_decode($x['body'],true); - - if (! $j) { - break; - } - - if (! is_array($j['item']) || ! count($j['item'])) { - break; - } - - Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]); - sleep($poll_interval); - - $page ++; - continue; - } - - $headers = [ - 'X-API-Token' => random_string(), - 'X-API-Request' => $hz_server . '/api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until), - 'Host' => $m['host'], - '(request-target)' => 'get /api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until), - ]; - - $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512'); - - $x = z_fetch_url($hz_server . '/api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until),false,$redirects,[ 'headers' => $headers ]); - - if (! $x['success']) { - logger('no API response'); - return; - } - - $j = json_decode($x['body'],true); - - if (! $j) { - return; - } - - if (! $j['success']) { - return; - } - - $poll_interval = get_config('system','poll_interval',3); - - if(count($j['results'])) { - $todo = count($j['results']); - logger('total to process: ' . $todo,LOGGER_DEBUG); - - foreach($j['results'] as $jj) { - Master::Summon([ 'File_importer',$jj['hash'], $channel['channel_address'], urlencode($hz_server) ]); - sleep($poll_interval); - } - } - - notice(t('Files and Posts imported.') . EOL); + Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]); + Master::Summon([ 'File_importer',sprintf('%d',$page), $channel['channel_address'], urlencode($hz_server) ]); } - notifications_on($channel['channel_id'], $saved_notification_flags); - - if (array_key_exists('item_id', $data) && $data['item_id']) - import_item_ids($channel, $data['item_id']); + // i do not think this is still used + //if (array_key_exists('item_id', $data) && $data['item_id']) + // import_item_ids($channel, $data['item_id']); // This will indirectly perform a refresh_all *and* update the directory Master::Summon(array('Directory', $channel['channel_id'])); - - notice(t('Import completed.') . EOL); - change_channel($channel['channel_id']); + notice(t('Import of items and files in progress') . EOL); + + // TODO: go away to a import progress page goaway(z_root()); } diff --git a/boot.php b/boot.php index cdba24f25..2145b30b1 100644 --- a/boot.php +++ b/boot.php @@ -1441,9 +1441,12 @@ function os_mkdir($path, $mode = 0777, $recursive = false) { */ function rrmdir($path) { if(is_dir($path) === true) { - $files = array_diff(scandir($path), array('.', '..')); - foreach($files as $file) { - rrmdir(realpath($path) . '/' . $file); + $dir_entries = scandir($path); + if (is_array($dir_entries)) { + $files = array_diff($dir_entries, array('.', '..')); + foreach($files as $file) { + rrmdir(realpath($path) . '/' . $file); + } } return rmdir($path); } diff --git a/include/attach.php b/include/attach.php index ef3d6e242..067da21ae 100644 --- a/include/attach.php +++ b/include/attach.php @@ -2442,7 +2442,6 @@ function attach_export_data($channel, $resource_id, $deleted = false, $zap_compa return $ret; } - /** * @brief Strip off 'store/nickname/' from the provided path * diff --git a/include/import.php b/include/import.php index ae7b2787a..f635f3705 100644 --- a/include/import.php +++ b/include/import.php @@ -1265,6 +1265,7 @@ function sync_files($channel, $files) { $store_path = $newfname; + $fp = fopen($newfname,'w'); if(! $fp) { logger('failed to open storage file.',LOGGER_NORMAL,LOG_ERR); -- cgit v1.2.3