From 3eeb2b0ee483bfb5c845dd4cc5e6a4915526a383 Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 27 Sep 2021 18:37:09 +0000 Subject: add importer daemons ported from zap --- Zotlabs/Daemon/Content_importer.php | 59 +++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 Zotlabs/Daemon/Content_importer.php (limited to 'Zotlabs/Daemon/Content_importer.php') diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php new file mode 100644 index 000000000..adc180a2c --- /dev/null +++ b/Zotlabs/Daemon/Content_importer.php @@ -0,0 +1,59 @@ + random_string(), + 'X-API-Request' => $hz_server . '/api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , + 'Host' => $m['host'], + '(request-target)' => 'get /api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , + ]; + + $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512'); + + $x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]); + + if(! $x['success']) { + logger('no API response',LOGGER_DEBUG); + killme(); + } + + $j = json_decode($x['body'],true); + + if (! $j) { + killme(); + } + + if(! ($j['item'] || count($j['item']))) + killme(); + + import_items($channel,$j['item'],false,((array_key_exists('relocate',$j)) ? $j['relocate'] : null)); + + killme(); + } +} -- cgit v1.2.3 From 5eb79bd51efbd69d297bbd6f46ee9d1583ec694e Mon Sep 17 00:00:00 2001 From: Mario Date: Mon, 27 Sep 2021 21:58:26 +0000 Subject: importer daemons: do not call with zap_compat flag and use return instead of kill so that queueworker can go on --- Zotlabs/Daemon/Content_importer.php | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'Zotlabs/Daemon/Content_importer.php') diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php index adc180a2c..72b8e34ce 100644 --- a/Zotlabs/Daemon/Content_importer.php +++ b/Zotlabs/Daemon/Content_importer.php @@ -24,19 +24,19 @@ class Content_importer { $channel = channelx_by_nick($channel_address); if(! $channel) { logger('itemhelper: channel not found'); - killme(); + return; } - $headers = [ + $headers = [ 'X-API-Token' => random_string(), - 'X-API-Request' => $hz_server . '/api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , + 'X-API-Request' => $hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , 'Host' => $m['host'], - '(request-target)' => 'get /api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , + '(request-target)' => 'get /api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page , ]; $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512'); - $x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&zap_compat=1&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]); + $x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]); if(! $x['success']) { logger('no API response',LOGGER_DEBUG); @@ -46,14 +46,14 @@ class Content_importer { $j = json_decode($x['body'],true); if (! $j) { - killme(); + return; } - if(! ($j['item'] || count($j['item']))) - killme(); + if(! is_array($j['item']) || ! count($j['item'])) + return; import_items($channel,$j['item'],false,((array_key_exists('relocate',$j)) ? $j['relocate'] : null)); - killme(); + return; } } -- cgit v1.2.3 From 4ba47698d7d0ef6c799bd0a28045c1d1e49c76f3 Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Wed, 29 Sep 2021 21:47:12 +0200 Subject: revise content import/export part 2 --- Zotlabs/Daemon/Content_importer.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'Zotlabs/Daemon/Content_importer.php') diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php index 72b8e34ce..ff6956051 100644 --- a/Zotlabs/Daemon/Content_importer.php +++ b/Zotlabs/Daemon/Content_importer.php @@ -23,7 +23,7 @@ class Content_importer { $channel = channelx_by_nick($channel_address); if(! $channel) { - logger('itemhelper: channel not found'); + logger('channel not found'); return; } @@ -44,7 +44,6 @@ class Content_importer { } $j = json_decode($x['body'],true); - if (! $j) { return; } @@ -52,8 +51,19 @@ class Content_importer { if(! is_array($j['item']) || ! count($j['item'])) return; + //$total_pages = floor(intval($j['items_total']) / intval($j['items_page'])); + //logger('importing items: ' . floor((intval($page) * 100) / $total_pages) . '%'); + + $saved_notification_flags = notifications_off($channel['channel_id']); + import_items($channel,$j['item'],false,((array_key_exists('relocate',$j)) ? $j['relocate'] : null)); + notifications_on($channel['channel_id'], $saved_notification_flags); + + $page++; + + Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]); + return; } } -- cgit v1.2.3 From b9b4e71f7d81cca23a08c55fc12db5f62ece2b56 Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Thu, 30 Sep 2021 12:25:03 +0200 Subject: provide a very simple status page for content import --- Zotlabs/Daemon/Content_importer.php | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'Zotlabs/Daemon/Content_importer.php') diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php index ff6956051..4a07b4cf4 100644 --- a/Zotlabs/Daemon/Content_importer.php +++ b/Zotlabs/Daemon/Content_importer.php @@ -3,6 +3,8 @@ namespace Zotlabs\Daemon; use Zotlabs\Web\HTTPSig; +use Zotlabs\Lib\PConfig; + require_once('include/cli_startup.php'); require_once('include/attach.php'); @@ -38,6 +40,8 @@ class Content_importer { $x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]); + // logger('item fetch: ' . print_r($x,true)); + if(! $x['success']) { logger('no API response',LOGGER_DEBUG); killme(); @@ -48,11 +52,9 @@ class Content_importer { return; } - if(! is_array($j['item']) || ! count($j['item'])) + if(! is_array($j['item']) || ! count($j['item'])) { return; - - //$total_pages = floor(intval($j['items_total']) / intval($j['items_page'])); - //logger('importing items: ' . floor((intval($page) * 100) / $total_pages) . '%'); + } $saved_notification_flags = notifications_off($channel['channel_id']); @@ -60,6 +62,14 @@ class Content_importer { notifications_on($channel['channel_id'], $saved_notification_flags); + PConfig::Set($channel['channel_id'], 'import', 'content_progress', [ + 'items_total' => $j['items_total'], + 'items_page' => $j['items_page'], + 'items_current_page' => count($j['item']), + 'last_page' => $page, + 'next_cmd' => ['Content_importer', sprintf('%d',$page + 1), $since, $until, $channel['channel_address'], urlencode($hz_server)] + ]); + $page++; Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]); -- cgit v1.2.3 From 47e83a15c1168cfd8ebc7905db853f3db6b3b31b Mon Sep 17 00:00:00 2001 From: Mario Vavti Date: Fri, 1 Oct 2021 21:52:30 +0200 Subject: import_progress: deal with the situation where items/files are being imported but there are none to import --- Zotlabs/Daemon/Content_importer.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'Zotlabs/Daemon/Content_importer.php') diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php index 4a07b4cf4..67f1c8e80 100644 --- a/Zotlabs/Daemon/Content_importer.php +++ b/Zotlabs/Daemon/Content_importer.php @@ -48,11 +48,9 @@ class Content_importer { } $j = json_decode($x['body'],true); - if (! $j) { - return; - } if(! is_array($j['item']) || ! count($j['item'])) { + PConfig::Set($channel['channel_id'], 'import', 'content_completed', 1); return; } -- cgit v1.2.3