aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Zotlabs/Daemon/Content_importer.php14
-rw-r--r--Zotlabs/Daemon/File_importer.php22
-rw-r--r--Zotlabs/Module/Import.php92
-rw-r--r--boot.php9
-rw-r--r--include/attach.php1
-rw-r--r--include/import.php1
6 files changed, 42 insertions, 97 deletions
diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php
index 72b8e34ce..ff6956051 100644
--- a/Zotlabs/Daemon/Content_importer.php
+++ b/Zotlabs/Daemon/Content_importer.php
@@ -23,7 +23,7 @@ class Content_importer {
$channel = channelx_by_nick($channel_address);
if(! $channel) {
- logger('itemhelper: channel not found');
+ logger('channel not found');
return;
}
@@ -44,7 +44,6 @@ class Content_importer {
}
$j = json_decode($x['body'],true);
-
if (! $j) {
return;
}
@@ -52,8 +51,19 @@ class Content_importer {
if(! is_array($j['item']) || ! count($j['item']))
return;
+ //$total_pages = floor(intval($j['items_total']) / intval($j['items_page']));
+ //logger('importing items: ' . floor((intval($page) * 100) / $total_pages) . '%');
+
+ $saved_notification_flags = notifications_off($channel['channel_id']);
+
import_items($channel,$j['item'],false,((array_key_exists('relocate',$j)) ? $j['relocate'] : null));
+ notifications_on($channel['channel_id'], $saved_notification_flags);
+
+ $page++;
+
+ Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]);
+
return;
}
}
diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php
index 883759a39..2be946fc3 100644
--- a/Zotlabs/Daemon/File_importer.php
+++ b/Zotlabs/Daemon/File_importer.php
@@ -14,7 +14,7 @@ class File_importer {
cli_startup();
- $attach_id = $argv[1];
+ $page = $argv[1];
$channel_address = $argv[2];
$hz_server = urldecode($argv[3]);
@@ -22,28 +22,36 @@ class File_importer {
$channel = channelx_by_nick($channel_address);
if(! $channel) {
- logger('filehelper: channel not found');
+ logger('channel not found');
return;
}
$headers = [
'X-API-Token' => random_string(),
- 'X-API-Request' => $hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id,
+ 'X-API-Request' => $hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page,
'Host' => $m['host'],
- '(request-target)' => 'get /api/z/1.0/file/export?f=&file_id=' . $attach_id,
+ '(request-target)' => 'get /api/z/1.0/file/export_page?f=records=1&page=' . $page,
];
$headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512');
- $x = z_fetch_url($hz_server . '/api/z/1.0/file/export?f=&file_id=' . $attach_id,false,$redirects,[ 'headers' => $headers ]);
+
+ $x = z_fetch_url($hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, false, $redirects, [ 'headers' => $headers ]);
if(! $x['success']) {
logger('no API response',LOGGER_DEBUG);
- return;
+ killme();
}
$j = json_decode($x['body'],true);
- $r = sync_files($channel,[$j]);
+ if(! is_array($j[0]['attach']) || ! count($j[0]['attach']))
+ return;
+
+ $r = sync_files($channel,$j);
+
+ $page++;
+
+ Master::Summon([ 'File_importer',sprintf('%d',$page), $channel['channel_address'], urlencode($hz_server) ]);
return;
}
diff --git a/Zotlabs/Module/Import.php b/Zotlabs/Module/Import.php
index ecbcc3034..a4697a426 100644
--- a/Zotlabs/Module/Import.php
+++ b/Zotlabs/Module/Import.php
@@ -509,8 +509,6 @@ class Import extends Controller {
$addon = array('channel' => $channel, 'data' => $data);
call_hooks('import_channel', $addon);
- $saved_notification_flags = notifications_off($channel['channel_id']);
-
if ($import_posts && array_key_exists('item', $data) && $data['item'])
import_items($channel, $data['item'], false, $relocate);
@@ -526,98 +524,24 @@ class Import extends Controller {
$poll_interval = get_config('system','poll_interval',3);
$page = 0;
- while (1) {
- $headers = [
- 'X-API-Token' => random_string(),
- 'X-API-Request' => $hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page ,
- 'Host' => $m['host'],
- '(request-target)' => 'get /api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page ,
- ];
-
- $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512');
-
- $x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]);
-
- // logger('z_fetch: ' . print_r($x,true));
-
- if (! $x['success']) {
- logger('no API response');
- break;
- }
-
- $j = json_decode($x['body'],true);
-
- if (! $j) {
- break;
- }
-
- if (! is_array($j['item']) || ! count($j['item'])) {
- break;
- }
-
- Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]);
- sleep($poll_interval);
-
- $page ++;
- continue;
- }
-
- $headers = [
- 'X-API-Token' => random_string(),
- 'X-API-Request' => $hz_server . '/api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until),
- 'Host' => $m['host'],
- '(request-target)' => 'get /api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until),
- ];
-
- $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'], channel_url($channel),true,'sha512');
-
- $x = z_fetch_url($hz_server . '/api/z/1.0/files?f=&since=' . urlencode($since) . '&until=' . urlencode($until),false,$redirects,[ 'headers' => $headers ]);
-
- if (! $x['success']) {
- logger('no API response');
- return;
- }
-
- $j = json_decode($x['body'],true);
-
- if (! $j) {
- return;
- }
-
- if (! $j['success']) {
- return;
- }
-
- $poll_interval = get_config('system','poll_interval',3);
-
- if(count($j['results'])) {
- $todo = count($j['results']);
- logger('total to process: ' . $todo,LOGGER_DEBUG);
-
- foreach($j['results'] as $jj) {
- Master::Summon([ 'File_importer',$jj['hash'], $channel['channel_address'], urlencode($hz_server) ]);
- sleep($poll_interval);
- }
- }
-
- notice(t('Files and Posts imported.') . EOL);
+ Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]);
+ Master::Summon([ 'File_importer',sprintf('%d',$page), $channel['channel_address'], urlencode($hz_server) ]);
}
- notifications_on($channel['channel_id'], $saved_notification_flags);
-
- if (array_key_exists('item_id', $data) && $data['item_id'])
- import_item_ids($channel, $data['item_id']);
+ // i do not think this is still used
+ //if (array_key_exists('item_id', $data) && $data['item_id'])
+ // import_item_ids($channel, $data['item_id']);
// This will indirectly perform a refresh_all *and* update the directory
Master::Summon(array('Directory', $channel['channel_id']));
-
- notice(t('Import completed.') . EOL);
-
change_channel($channel['channel_id']);
+ notice(t('Import of items and files in progress') . EOL);
+
+ // TODO: go away to a import progress page
goaway(z_root());
}
diff --git a/boot.php b/boot.php
index cdba24f25..2145b30b1 100644
--- a/boot.php
+++ b/boot.php
@@ -1441,9 +1441,12 @@ function os_mkdir($path, $mode = 0777, $recursive = false) {
*/
function rrmdir($path) {
if(is_dir($path) === true) {
- $files = array_diff(scandir($path), array('.', '..'));
- foreach($files as $file) {
- rrmdir(realpath($path) . '/' . $file);
+ $dir_entries = scandir($path);
+ if (is_array($dir_entries)) {
+ $files = array_diff($dir_entries, array('.', '..'));
+ foreach($files as $file) {
+ rrmdir(realpath($path) . '/' . $file);
+ }
}
return rmdir($path);
}
diff --git a/include/attach.php b/include/attach.php
index ef3d6e242..067da21ae 100644
--- a/include/attach.php
+++ b/include/attach.php
@@ -2442,7 +2442,6 @@ function attach_export_data($channel, $resource_id, $deleted = false, $zap_compa
return $ret;
}
-
/**
* @brief Strip off 'store/nickname/' from the provided path
*
diff --git a/include/import.php b/include/import.php
index ae7b2787a..f635f3705 100644
--- a/include/import.php
+++ b/include/import.php
@@ -1265,6 +1265,7 @@ function sync_files($channel, $files) {
$store_path = $newfname;
+
$fp = fopen($newfname,'w');
if(! $fp) {
logger('failed to open storage file.',LOGGER_NORMAL,LOG_ERR);
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-24 21:45:33 +0000