diff options
| -rw-r--r-- | Zotlabs/Daemon/Channel_purge.php | 34 | ||||
| -rw-r--r-- | Zotlabs/Daemon/Content_importer.php | 18 | ||||
| -rw-r--r-- | Zotlabs/Daemon/File_importer.php | 16 | ||||
| -rw-r--r-- | Zotlabs/Module/Import.php | 23 | ||||
| -rw-r--r-- | Zotlabs/Module/Import_progress.php | 67 | ||||
| -rw-r--r-- | include/api_zot.php | 22 | ||||
| -rw-r--r-- | include/channel.php | 12 | ||||
| -rw-r--r-- | tests/unit/includes/MarkdownTest.php | 4 |
8 files changed, 170 insertions, 26 deletions
diff --git a/Zotlabs/Daemon/Channel_purge.php b/Zotlabs/Daemon/Channel_purge.php new file mode 100644 index 000000000..416126896 --- /dev/null +++ b/Zotlabs/Daemon/Channel_purge.php @@ -0,0 +1,34 @@ +<?php + +namespace Zotlabs\Daemon; + +class Channel_purge { + + static public function run($argc,$argv) { + + cli_startup(); + + $channel_id = intval($argv[1]); + + $channel = q("select * from channel where channel_id = %d and channel_removed = 1", + intval($channel_id) + ); + + if (! $channel) { + return; + } + + do { + $r = q("select id from item where uid = %d and item_deleted = 0 limit 100", + intval($channel_id) + ); + if ($r) { + foreach ($r as $rv) { + drop_item($rv['id'],false); + } + } + } while ($r); + + return; + } +} diff --git a/Zotlabs/Daemon/Content_importer.php b/Zotlabs/Daemon/Content_importer.php index ff6956051..4a07b4cf4 100644 --- a/Zotlabs/Daemon/Content_importer.php +++ b/Zotlabs/Daemon/Content_importer.php @@ -3,6 +3,8 @@ namespace Zotlabs\Daemon; use Zotlabs\Web\HTTPSig; +use Zotlabs\Lib\PConfig; + require_once('include/cli_startup.php'); require_once('include/attach.php'); @@ -38,6 +40,8 @@ class Content_importer { $x = z_fetch_url($hz_server . '/api/z/1.0/item/export_page?f=&since=' . urlencode($since) . '&until=' . urlencode($until) . '&page=' . $page,false,$redirects,[ 'headers' => $headers ]); + // logger('item fetch: ' . print_r($x,true)); + if(! $x['success']) { logger('no API response',LOGGER_DEBUG); killme(); @@ -48,11 +52,9 @@ class Content_importer { return; } - if(! is_array($j['item']) || ! count($j['item'])) + if(! is_array($j['item']) || ! count($j['item'])) { return; - - //$total_pages = floor(intval($j['items_total']) / intval($j['items_page'])); - //logger('importing items: ' . floor((intval($page) * 100) / $total_pages) . '%'); + } $saved_notification_flags = notifications_off($channel['channel_id']); @@ -60,6 +62,14 @@ class Content_importer { notifications_on($channel['channel_id'], $saved_notification_flags); + PConfig::Set($channel['channel_id'], 'import', 'content_progress', [ + 'items_total' => $j['items_total'], + 'items_page' => $j['items_page'], + 'items_current_page' => count($j['item']), + 'last_page' => $page, + 'next_cmd' => ['Content_importer', sprintf('%d',$page + 1), $since, $until, $channel['channel_address'], urlencode($hz_server)] + ]); + $page++; Master::Summon([ 'Content_importer', sprintf('%d',$page), $since, $until, $channel['channel_address'], urlencode($hz_server) ]); diff --git a/Zotlabs/Daemon/File_importer.php b/Zotlabs/Daemon/File_importer.php index 2be946fc3..28fb172a7 100644 --- a/Zotlabs/Daemon/File_importer.php +++ b/Zotlabs/Daemon/File_importer.php @@ -3,6 +3,8 @@ namespace Zotlabs\Daemon; use Zotlabs\Web\HTTPSig; +use Zotlabs\Lib\PConfig; + require_once('include/cli_startup.php'); require_once('include/attach.php'); @@ -35,7 +37,9 @@ class File_importer { $headers = HTTPSig::create_sig($headers,$channel['channel_prvkey'],channel_url($channel),true,'sha512'); + // TODO: implement total count $x = z_fetch_url($hz_server . '/api/z/1.0/file/export_page?f=records=1&page=' . $page, false, $redirects, [ 'headers' => $headers ]); + // logger('file fetch: ' . print_r($x,true)); if(! $x['success']) { logger('no API response',LOGGER_DEBUG); @@ -44,10 +48,18 @@ class File_importer { $j = json_decode($x['body'],true); - if(! is_array($j[0]['attach']) || ! count($j[0]['attach'])) + if(! is_array($j['results'][0]['attach']) || ! count($j['results'][0]['attach'])) { return; + } + + $r = sync_files($channel, $j['results']); - $r = sync_files($channel,$j); + PConfig::Set($channel['channel_id'], 'import', 'files_progress', [ + 'files_total' => $j['total'], + 'files_page' => 1, // export page atm returns just one file + 'last_page' => $page, + 'next_cmd' => ['File_importer',sprintf('%d',$page + 1), $channel['channel_address'], urlencode($hz_server)] + ]); $page++; diff --git a/Zotlabs/Module/Import.php b/Zotlabs/Module/Import.php index a4697a426..38b64dd3c 100644 --- a/Zotlabs/Module/Import.php +++ b/Zotlabs/Module/Import.php @@ -12,6 +12,7 @@ use Zotlabs\Daemon\Master; use Zotlabs\Lib\Libzot; use Zotlabs\Web\Controller; use Zotlabs\Web\HTTPSig; +use Zotlabs\Lib\PConfig; /** @@ -435,6 +436,7 @@ class Import extends Controller { logger('import step 8'); } + // import groups $groups = $data['group']; if ($groups) { @@ -476,6 +478,7 @@ class Import extends Controller { logger('import step 9'); + if (is_array($data['obj'])) import_objs($channel, $data['obj']); @@ -509,8 +512,15 @@ class Import extends Controller { $addon = array('channel' => $channel, 'data' => $data); call_hooks('import_channel', $addon); - if ($import_posts && array_key_exists('item', $data) && $data['item']) + if ($import_posts && array_key_exists('item', $data) && $data['item']) { import_items($channel, $data['item'], false, $relocate); + } + + // Immediately notify old server about the new clone + Master::Summon( [ 'Notifier', 'refresh_all', $channel['channel_id'] ] ); + + // This will indirectly perform a refresh_all *and* update the directory + Master::Summon(array('Directory', $channel['channel_id'])); if ($api_path && $import_posts) { // we are importing from a server and not a file @@ -533,16 +543,15 @@ class Import extends Controller { //if (array_key_exists('item_id', $data) && $data['item_id']) // import_item_ids($channel, $data['item_id']); - // This will indirectly perform a refresh_all *and* update the directory - - Master::Summon(array('Directory', $channel['channel_id'])); - change_channel($channel['channel_id']); - notice(t('Import of items and files in progress') . EOL); + notice(t('Content import in progress...') . EOL); + + if ($api_path && $import_posts) + goaway(z_root() . '/import_progress'); - // TODO: go away to a import progress page goaway(z_root()); + } /** diff --git a/Zotlabs/Module/Import_progress.php b/Zotlabs/Module/Import_progress.php new file mode 100644 index 000000000..f255dcbe9 --- /dev/null +++ b/Zotlabs/Module/Import_progress.php @@ -0,0 +1,67 @@ +<?php +namespace Zotlabs\Module; + +use Zotlabs\Lib\PConfig; + +class Import_progress extends \Zotlabs\Web\Controller { + + function post() { + + if(! local_channel()) + return; + + } + + function get() { + + if(! local_channel()) { + return; + } + + // items + $c = PConfig::Get(local_channel(), 'import', 'content_progress'); + + if (!$c) { + $co = 'Item sync status: waiting to start...'; + } + else { + $total_cpages = floor(intval($c['items_total']) / intval($c['items_page'])); + if(!$total_cpages) + $total_cpages = 1; // because of floor + + $cpage = $c['last_page'] + 1; // because page count start at 0 + + $co = 'Item sync status: ' . floor((intval($cpage) * 100) / $total_cpages) . '%'; + } + + // files + $f = PConfig::Get(local_channel(), 'import', 'files_progress'); + + if (!$f) { + $fo = 'File sync status: waiting to start...'; + } + else { + $total_fpages = floor(intval($f['files_total']) / intval($f['files_page'])); + if(!$total_fpages) + $total_fpages = 1; + + $fpage = $f['last_page'] + 1; + + $fo = 'File sync status: ' . floor((intval($fpage) * 100) / $total_fpages) . '%'; + } + + $o .= '<h3>' . $co . '</h3>'; + if (is_array($c)) + $o .= '<pre>' . htmlspecialchars(print_array($c)) . '</pre>'; + + $o .= '<h3>' . $fo . '</h3>'; + if (is_array($f)) + $o .= '<pre>' . htmlspecialchars(print_array($f)) . '</pre>'; + + $o .= '<hr>'; + $o .= '<h3>Refresh page for updates!</h3>'; + + return $o; + } + +} diff --git a/include/api_zot.php b/include/api_zot.php index 93b99c0c8..7a217854f 100644 --- a/include/api_zot.php +++ b/include/api_zot.php @@ -305,24 +305,38 @@ if(api_user() === false) return false; - $ret = []; $codebase = ((isset($_REQUEST['zap_compat']) && $_REQUEST['zap_compat']) ? true : false); $channel = channelx_by_n(api_user()); $page = ((array_key_exists('page',$_REQUEST)) ? intval($_REQUEST['page']) : 0); + $ret['success'] = false; + $ret['total'] = 0; + $ret['results'] = []; + + $count = attach_count_files($channel['channel_id'], get_observer_hash()); + if($count['success']) { + $ret['total'] = $count['results']; + } + + if (!$ret['total']) { + json_return_and_die($ret); + } + $files = attach_list_files($channel['channel_id'], get_observer_hash(), '', '', '', 'created asc' ,$page, 1); if (!$files['success']) { - return false; + json_return_and_die($ret); } foreach($files['results'] as $file) { - $ret[] = attach_export_data($channel, $file['hash'], false, $codebase); + $ret['results'][] = attach_export_data($channel, $file['hash'], false, $codebase); } - if($ret) { + if($ret['results']) { + $ret['success'] = true; json_return_and_die($ret); } + killme(); } diff --git a/include/channel.php b/include/channel.php index 53b781eb6..ca6d01498 100644 --- a/include/channel.php +++ b/include/channel.php @@ -2825,13 +2825,6 @@ function channel_remove($channel_id, $local = true, $unset_session = false) { } } - $r = q("select id from item where uid = %d", intval($channel_id)); - if($r) { - foreach($r as $rv) { - drop_item($rv['id'],false); - } - } - q("delete from abook where abook_xchan = '%s' and abook_self = 1 ", dbesc($channel['channel_hash']) ); @@ -2841,6 +2834,9 @@ function channel_remove($channel_id, $local = true, $unset_session = false) { intval($channel_id) ); + // remove items + Master::Summon([ 'Channel_purge', $channel_id ]); + // if this was the default channel, set another one as default if(App::$account['account_default_channel'] == $channel_id) { $r = q("select channel_id from channel where channel_account_id = %d and channel_removed = 0 limit 1", @@ -2881,6 +2877,8 @@ function channel_remove($channel_id, $local = true, $unset_session = false) { $r = q("UPDATE xchan SET xchan_deleted = 1 WHERE xchan_hash = '%s'", dbesc($channel['channel_hash']) ); + // send a cleanup message to other servers + Master::Summon([ 'Notifier', 'purge_all', $channel_id ]); } //remove from file system diff --git a/tests/unit/includes/MarkdownTest.php b/tests/unit/includes/MarkdownTest.php index 2a92a58d2..98fe40a49 100644 --- a/tests/unit/includes/MarkdownTest.php +++ b/tests/unit/includes/MarkdownTest.php @@ -87,8 +87,8 @@ class MarkdownTest extends UnitTestCase { "1. Item 1\n2. Item 2\n3. Item **3**" ], 'nested lists' => [ - '<ul><li>Item 1<ol><li>Item 1a</li><li>Item <b>1b</b></ol></li><li>Item 2</li></ul>', - "- Item 1\n 1. Item 1a\n 2. Item **1b**\n- Item 2" + '<ul><li>Item A</li><li>Item B<ul><li>Nested A</li><li>Nested B</li></ul></li><li>Item C</li></ul>', + "- Item A\n- Item B\n - Nested A\n - Nested B\n- Item C" ], 'img' => [ '<img src="/path/to/img.png" alt="alt text" title="title text">', |