From 3eeb2b0ee483bfb5c845dd4cc5e6a4915526a383 Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 27 Sep 2021 18:37:09 +0000 Subject: add importer daemons ported from zap --- Zotlabs/Daemon/File_importer.php | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 Zotlabs/Daemon/File_importer.php (limited to 'Zotlabs/Daemon/File_importer.php') diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php new file mode 100644 index 000000000..f3fe785bb --- /dev/null +++ b/Zotlabs/Daemon/File_importer.php @@ -0,0 +1,50 @@ + random_string(), + 'X-API-Request' => $hz_server . '/api/z/1.0/file/export?f=&zap_compat=1&file_id=' . $attach_id, + 'Host' => $m['host'], + '(request-target)' => 'get /api/z/1.0/file/export?f=&zap_compat=1&file_id=' . $attach_id, + ]; + + $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512'); + $x = z_fetch_url($hz_server . '/api/z/1.0/file/export?f=&zap_compat=1&file_id=' . $attach_id,false,$redirects,[ 'headers' => $headers ]); + + if(! $x['success']) { + logger('no API response',LOGGER_DEBUG); + return; + } + + $j = json_decode($x['body'],true); + + $r = sync_files($channel,[$j]); + + killme(); + } +} -- cgit v1.2.3 From 5eb79bd51efbd69d297bbd6f46ee9d1583ec694e Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 27 Sep 2021 21:58:26 +0000 Subject: importer daemons: do not call with zap_compat flag and use return instead of kill so that queueworker can go on --- Zotlabs/Daemon/File_importer.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'Zotlabs/Daemon/File_importer.php') diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php index f3fe785bb..883759a39 100644 --- a/Zotlabs/Daemon/File_importer.php +++ b/Zotlabs/Daemon/File_importer.php @@ -23,18 +23,18 @@ class File_importer { $channel = channelx_by_nick($channel_address); if(! $channel) { logger('filehelper: channel not found'); - killme(); + return; } $headers = [ 'X-API-Token' => random_string(), - 'X-API-Request' => $hz_server . '/api/z/1.0/file/export?f=&zap_compat=1&file_id=' . $attach_id, + 'X-API-Request' => $hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id, 'Host' => $m['host'], - '(request-target)' => 'get /api/z/1.0/file/export?f=&zap_compat=1&file_id=' . $attach_id, + '(request-target)' => 'get /api/z/1.0/file/export?f=&file_id=' . $attach_id, ]; - $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512'); - $x = z_fetch_url($hz_server . '/api/z/1.0/file/export?f=&zap_compat=1&file_id=' . $attach_id,false,$redirects,[ 'headers' => $headers ]); + $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512'); + $x = z_fetch_url($hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id,false,$redirects,[ 'headers' => $headers ]); if(! $x['success']) { logger('no API response',LOGGER_DEBUG); @@ -45,6 +45,6 @@ class File_importer { $r = sync_files($channel,[$j]); - killme(); + return; } } -- cgit v1.2.3 From 4ba47698d7d0ef6c799bd0a28045c1d1e49c76f3 Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Wed, 29 Sep 2021 21:47:12 +0200 Subject: revise content import/export part 2 --- Zotlabs/Daemon/File_importer.php | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'Zotlabs/Daemon/File_importer.php') diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php index 883759a39..2be946fc3 100644 --- a/Zotlabs/Daemon/File_importer.php +++ b/Zotlabs/Daemon/File_importer.php @@ -14,7 +14,7 @@ class File_importer { cli_startup(); - $attach_id = $argv[1]; + $page = $argv[1]; $channel_address = $argv[2]; $hz_server = urldecode($argv[3]); @@ -22,28 +22,36 @@ class File_importer { $channel = channelx_by_nick($channel_address); if(! $channel) { - logger('filehelper: channel not found'); + logger('channel not found'); return; } $headers = [ 'X-API-Token' => random_string(), - 'X-API-Request' => $hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id, + 'X-API-Request' => $hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, 'Host' => $m['host'], - '(request-target)' => 'get /api/z/1.0/file/export?f=&file_id=' . $attach_id, + '(request-target)' => 'get /api/z/1.0/file/export_page?f=records=1&page=' . $page, ]; $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512'); - $x = z_fetch_url($hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id,false,$redirects,[ 'headers' => $headers ]); + + $x = z_fetch_url($hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, false, $redirects, [ 'headers' => $headers ]); if(! $x['success']) { logger('no API response',LOGGER_DEBUG); - return; + killme(); } $j = json_decode($x['body'],true); - $r = sync_files($channel,[$j]); + if(! is_array($j[0]['attach']) || ! count($j[0]['attach'])) + return; + + $r = sync_files($channel,$j); + + $page++; + + Master::Summon([ 'File_importer',sprintf('%d',$page), $channel['channel_address'], urlencode($hz_server) ]); return; } -- cgit v1.2.3 From b9b4e71f7d81cca23a08c55fc12db5f62ece2b56 Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Thu, 30 Sep 2021 12:25:03 +0200 Subject: provide a very simple status page for content import --- Zotlabs/Daemon/File_importer.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'Zotlabs/Daemon/File_importer.php') diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php index 2be946fc3..bb12cdf70 100644 --- a/Zotlabs/Daemon/File_importer.php +++ b/Zotlabs/Daemon/File_importer.php @@ -3,6 +3,8 @@ namespace Zotlabs\Daemon; use Zotlabs\Web\HTTPSig; +use Zotlabs\Lib\PConfig; + require_once('include/cli_startup.php'); require_once('include/attach.php'); @@ -35,7 +37,9 @@ class File_importer { $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512'); + // TODO: implement total count $x = z_fetch_url($hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, false, $redirects, [ 'headers' => $headers ]); + // logger('file fetch: ' . print_r($x,true)); if(! $x['success']) { logger('no API response',LOGGER_DEBUG); @@ -44,8 +48,10 @@ class File_importer { $j = json_decode($x['body'],true); - if(! is_array($j[0]['attach']) || ! count($j[0]['attach'])) + if(! is_array($j[0]['attach']) || ! count($j[0]['attach'])) { + PConfig::Set($channel['channel_id'], 'import', 'files', 1); return; + } $r = sync_files($channel,$j); -- cgit v1.2.3 From 6236869ebed317aa06078f606156edcec9f9b55b Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Thu, 30 Sep 2021 17:34:30 +0200 Subject: implement file totals and add mod import_progress --- Zotlabs/Daemon/File_importer.php | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'Zotlabs/Daemon/File_importer.php') diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php index bb12cdf70..28fb172a7 100644 --- a/Zotlabs/Daemon/File_importer.php +++ b/Zotlabs/Daemon/File_importer.php @@ -48,12 +48,18 @@ class File_importer { $j = json_decode($x['body'],true); - if(! is_array($j[0]['attach']) || ! count($j[0]['attach'])) { - PConfig::Set($channel['channel_id'], 'import', 'files', 1); + if(! is_array($j['results'][0]['attach']) || ! count($j['results'][0]['attach'])) { return; } - $r = sync_files($channel,$j); + $r = sync_files($channel, $j['results']); + + PConfig::Set($channel['channel_id'], 'import', 'files_progress', [ + 'files_total' => $j['total'], + 'files_page' => 1, // export page atm returns just one file + 'last_page' => $page, + 'next_cmd' => ['File_importer',sprintf('%d',$page + 1), $channel['channel_address'], urlencode($hz_server)] + ]); $page++; -- cgit v1.2.3 From 47e83a15c1168cfd8ebc7905db853f3db6b3b31b Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Fri, 1 Oct 2021 21:52:30 +0200 Subject: import_progress: deal with the situation where items/files are being imported but there are none to import --- Zotlabs/Daemon/File_importer.php | 1 + 1 file changed, 1 insertion(+) (limited to 'Zotlabs/Daemon/File_importer.php') diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php index 28fb172a7..7067e152d 100644 --- a/Zotlabs/Daemon/File_importer.php +++ b/Zotlabs/Daemon/File_importer.php @@ -49,6 +49,7 @@ class File_importer { $j = json_decode($x['body'],true); if(! is_array($j['results'][0]['attach']) || ! count($j['results'][0]['attach'])) { + PConfig::Set($channel['channel_id'], 'import', 'files_completed', 1); return; } -- cgit v1.2.3