Refactored the scan and import functions to reduce redundant code and simplify logic. Import of pages, layouts, and blocks works.

1 files changed, 23 insertions, 373 deletions

diff --git a/Zotlabs/Module/Webpages.php b/Zotlabs/Module/Webpages.php
index 1c9586615..766b4fc09 100644
--- a/Zotlabs/Module/Webpages.php
+++ b/Zotlabs/Module/Webpages.php
@@ -231,383 +231,33 @@ class Webpages extends \Zotlabs\Web\Controller {
 				$website = dirname($source) . '/' . $tmp_folder_name;
 				$zip->extractTo($website); // change this to the correct site path
 				$zip->close();
-				@unlink($source);
+				@unlink($source);	// delete the compressed file now that the content has been extracted
 
-				$hubsites = $this->import_website($website);
+				require_once('include/import.php');
+				$elements = [];
+				$elements['pages'] = scan_webpage_elements($website, 'page');
+				$elements['layouts'] = scan_webpage_elements($website, 'layout');
+				$elements['blocks'] = scan_webpage_elements($website, 'block');
+				
 				$channel = \App::get_channel();
-				$blocks = $this->import_blocks($channel, $hubsites['blocks']);
-				$pages = $this->import_pages($channel, $hubsites['pages']);
-				$layouts = $this->import_layouts($channel, $hubsites['layouts']);
-				if($blocks || $pages || $layouts) {	// Without the if statement, the folder is deleted before the import_blocks function completes.
-					rrmdir($website);
+				// Import layout first so that pages that reference new layouts will find
+				// the mid of layout items in the database
+				foreach($elements['layouts'] as &$layout) {
+					$layout = import_webpage_element($layout, $channel, 'layout');
 				}
-			}
-			
-		
+				foreach($elements['pages'] as &$page) {
+					$page = import_webpage_element($page, $channel, 'page');
+				}
+				foreach($elements['blocks'] as &$block) {
+					$block = import_webpage_element($block, $channel, 'block');
+				}
+				
+				// Without the if statement below, the folder is deleted before the import completes.
+				if($elements) {	
+					rrmdir($website);	// Delete the temporary decompressed files
+				}
+			}		
 		}
 	}
 	
-	private function import_website($path) {
-			$hubsites = [];
-			$pages = [];
-			$blocks = [];
-			$layouts = [];
-			// Import pages
-			$dirtoscan = $path . '/pages/';
-			if (is_dir($dirtoscan)) {
-					$dirlist = scandir($dirtoscan);
-					if ($dirlist) {
-							foreach ($dirlist as $element) {
-									if ($element === '.' || $element === '..') {
-											continue;
-									}
-									$folder = $dirtoscan . '/' . $element;
-									if (is_dir($folder)) {
-											$jsonfilepath = $folder . '/page.json';
-											if (is_file($jsonfilepath)) {
-													$pagejson = json_decode(file_get_contents($jsonfilepath), true);
-													$pagejson['path'] = $folder . '/' . $pagejson['contentfile'];
-													if ($pagejson['contentfile'] === '') {
-															logger('Invalid page content file');
-															return false;
-													}
-													$pagecontent = file_get_contents($folder . '/' . $pagejson['contentfile']);
-													if (!$pagecontent) {
-															logger('Failed to get file content for ' . $pagejson['contentfile']);
-															return false;
-													}
-													$pages[] = $pagejson;
-											}
-									}
-							}
-					}
-			}
-			$hubsites['pages'] = $pages;
-			// Import layouts
-			$dirtoscan = $path . '/layouts/';
-			if (is_dir($dirtoscan)) {
-					$dirlist = scandir($dirtoscan);
-					if ($dirlist) {
-							foreach ($dirlist as $element) {
-									if ($element === '.' || $element === '..') {
-											continue;
-									}
-									$folder = $dirtoscan . '/' . $element;
-									if (is_dir($folder)) {
-											$jsonfilepath = $folder . '/layout.json';
-											if (is_file($jsonfilepath)) {
-													$layoutjson = json_decode(file_get_contents($jsonfilepath), true);
-													$layoutjson['path'] = $folder . '/' . $layoutjson['contentfile'];
-													if ($layoutjson['contentfile'] === '') {
-															logger('Invalid layout content file');
-															return false;
-													}
-													$layoutcontent = file_get_contents($folder . '/' . $layoutjson['contentfile']);
-													if (!$layoutcontent) {
-															logger('Failed to get file content for ' . $layoutjson['contentfile']);
-															return false;
-													}
-													$layouts[] = $layoutjson;
-											}
-									}
-							}
-					}
-			}
-			$hubsites['layouts'] = $layouts;
-			// Import blocks
-			$dirtoscan = $path . '/blocks/';
-			if (is_dir($dirtoscan)) {
-					$dirlist = scandir($dirtoscan);
-					if ($dirlist) {
-							foreach ($dirlist as $element) {
-									if ($element === '.' || $element === '..') {
-											continue;
-									}
-									$folder = $dirtoscan . '/' . $element;
-									if (is_dir($folder)) {
-											$jsonfilepath = $folder . '/block.json';
-											if (is_file($jsonfilepath)) {
-													$block = json_decode(file_get_contents($jsonfilepath), true);
-													$block['path'] = $folder . '/' . $block['contentfile'];
-													if ($block['contentfile'] === '') {
-															logger('Invalid block content file');
-															return false;
-													}
-													$blockcontent = file_get_contents($folder . '/' . $block['contentfile']);
-													if (!$blockcontent) {
-															logger('Failed to get file content for ' . $block['contentfile']);
-															return false;
-													}
-													$blocks[] = $block;
-											}
-									}
-							}
-					}
-			}
-			$hubsites['blocks'] = $blocks;
-			return $hubsites;
-	}
-	
-	private function import_blocks($channel, $blocks) {
-    foreach ($blocks as &$b) {
-        
-        $arr = array();
-        $arr['item_type'] = ITEM_TYPE_BLOCK;
-        $namespace = 'BUILDBLOCK';
-        $arr['uid'] = $channel['channel_id'];
-        $arr['aid'] = $channel['channel_account_id'];
-        
-        $iid = q("select iid from iconfig where k = 'BUILDBLOCK' and v = '%s' and cat = 'system'",
-                dbesc($b['name'])
-        );
-        if($iid) {
-            $iteminfo = q("select mid,created,edited from item where id = %d",
-                    intval($iid[0]['iid'])
-            );
-            $arr['mid'] = $arr['parent_mid'] = $iteminfo[0]['mid'];
-            $arr['created'] = $iteminfo[0]['created'];
-            $arr['edited'] = (($b['edited']) ? datetime_convert('UTC', 'UTC', $b['edited']) : datetime_convert());
-        } else {
-            $arr['created'] = (($b['created']) ? datetime_convert('UTC', 'UTC', $b['created']) : datetime_convert());
-            $arr['edited'] = datetime_convert('UTC', 'UTC', '0000-00-00 00:00:00');
-            $arr['mid'] = $arr['parent_mid'] = item_message_id();
-        }
-        $arr['title'] = $b['title'];
-        $arr['body'] = file_get_contents($b['path']);
-        $arr['owner_xchan'] = get_observer_hash();
-        $arr['author_xchan'] = (($b['author_xchan']) ? $b['author_xchan'] : get_observer_hash());
-        if(($b['mimetype'] === 'text/bbcode' || $b['mimetype'] === 'text/html' ||
-                $b['mimetype'] === 'text/markdown' ||$b['mimetype'] === 'text/plain' ||
-                $b['mimetype'] === 'application/x-pdl' ||$b['mimetype'] === 'application/x-php')) {
-            $arr['mimetype'] = $b['mimetype'];
-        } else {
-            $arr['mimetype'] = 'text/bbcode';
-        }
-
-        $pagetitle = $b['name']; 
-
-        // Verify ability to use html or php!!!
-        $execflag = false;
-        if ($arr['mimetype'] === 'application/x-php') {
-            $z = q("select account_id, account_roles, channel_pageflags from account left join channel on channel_account_id = account_id where channel_id = %d limit 1", intval(local_channel())
-            );
-
-            if ($z && (($z[0]['account_roles'] & ACCOUNT_ROLE_ALLOWCODE) || ($z[0]['channel_pageflags'] & PAGE_ALLOWCODE))) {
-                $execflag = true;
-            }
-        }
-
-        $remote_id = 0;
-
-        $z = q("select * from iconfig where v = '%s' and k = '%s' and cat = 'service' limit 1", dbesc($pagetitle), dbesc($namespace));
-
-        $i = q("select id, edited, item_deleted from item where mid = '%s' and uid = %d limit 1", dbesc($arr['mid']), intval(local_channel())
-        );
-        if ($z && $i) {
-            $remote_id = $z[0]['id'];
-            $arr['id'] = $i[0]['id'];
-            // don't update if it has the same timestamp as the original
-            if ($arr['edited'] > $i[0]['edited'])
-                $x = item_store_update($arr, $execflag);
-        } else {
-            if (($i) && (intval($i[0]['item_deleted']))) {
-                // was partially deleted already, finish it off
-                q("delete from item where mid = '%s' and uid = %d", dbesc($arr['mid']), intval(local_channel())
-                );
-            }
-            $x = item_store($arr, $execflag);
-        }
-        if ($x['success']) {
-            $item_id = $x['item_id'];
-            update_remote_id($channel, $item_id, $arr['item_type'], $pagetitle, $namespace, $remote_id, $arr['mid']);
-            $b['import_success'] = 1;
-        } else {
-            $b['import_success'] = 0;
-        }
-    }
-    return $blocks;
-}
-
-
-private function import_pages($channel, $pages) {
-    foreach ($pages as &$p) {
-        
-        $arr = array();
-        $arr['item_type'] = ITEM_TYPE_WEBPAGE;
-        $namespace = 'WEBPAGE';
-        $arr['uid'] = $channel['channel_id'];
-        $arr['aid'] = $channel['channel_account_id'];
-
-        if($p['pagelink']) {
-                require_once('library/urlify/URLify.php');
-                $pagetitle = strtolower(\URLify::transliterate($p['pagelink']));
-        }
-        $arr['layout_mid'] = ''; // by default there is no layout associated with the page
-        // If a layout was specified, find it in the database and get its info. If
-        // it does not exist, leave layout_mid empty
-        logger('hubsites plugin: $p[layout] = ' . $p['layout']);
-        if($p['layout'] !== '') {
-            $liid = q("select iid from iconfig where k = 'PDL' and v = '%s' and cat = 'system'",
-                    dbesc($p['layout'])
-            );
-            if($liid) {
-                $linfo = q("select mid from item where id = %d",
-                        intval($liid[0]['iid'])
-                );
-                logger('hubsites plugin: $linfo= ' . json_encode($linfo,true));
-                $arr['layout_mid'] = $linfo[0]['mid'];
-            }                 
-        }
-        // See if the page already exists
-        $iid = q("select iid from iconfig where k = 'WEBPAGE' and v = '%s' and cat = 'system'",
-                dbesc($pagetitle)
-        );
-        if($iid) {
-            // Get the existing page info
-            $pageinfo = q("select mid,layout_mid,created,edited from item where id = %d",
-                    intval($iid[0]['iid'])
-            );
-            $arr['mid'] = $arr['parent_mid'] = $pageinfo[0]['mid'];
-            $arr['created'] = $pageinfo[0]['created'];
-            $arr['edited'] = (($p['edited']) ? datetime_convert('UTC', 'UTC', $p['edited']) : datetime_convert());
-        } else {
-            $arr['created'] = (($p['created']) ? datetime_convert('UTC', 'UTC', $p['created']) : datetime_convert());
-            $arr['edited'] = datetime_convert('UTC', 'UTC', '0000-00-00 00:00:00');
-            $arr['mid'] = $arr['parent_mid'] = item_message_id();
-        }
-        $arr['title'] = $p['title'];
-        $arr['body'] = file_get_contents($p['path']);
-        $arr['term'] = $p['term'];  // Not sure what this is supposed to be
-        
-        $arr['owner_xchan'] = get_observer_hash();
-        $arr['author_xchan'] = (($p['author_xchan']) ? $p['author_xchan'] : get_observer_hash());
-        if(($p['mimetype'] === 'text/bbcode' || $p['mimetype'] === 'text/html' ||
-                $p['mimetype'] === 'text/markdown' ||$p['mimetype'] === 'text/plain' ||
-                $p['mimetype'] === 'application/x-pdl' ||$p['mimetype'] === 'application/x-php')) {
-            $arr['mimetype'] = $p['mimetype'];
-        } else {
-            $arr['mimetype'] = 'text/bbcode';
-        }
-
-        // Verify ability to use html or php!!!
-        $execflag = false;
-        if ($arr['mimetype'] === 'application/x-php') {
-            $z = q("select account_id, account_roles, channel_pageflags from account left join channel on channel_account_id = account_id where channel_id = %d limit 1", intval(local_channel())
-            );
-
-            if ($z && (($z[0]['account_roles'] & ACCOUNT_ROLE_ALLOWCODE) || ($z[0]['channel_pageflags'] & PAGE_ALLOWCODE))) {
-                $execflag = true;
-            }
-        }
-
-        $remote_id = 0;
-
-        $z = q("select * from iconfig where v = '%s' and k = '%s' and cat = 'system' limit 1",
-                dbesc($pagetitle),
-                dbesc($namespace)
-        );
-
-        $i = q("select id, edited, item_deleted from item where mid = '%s' and uid = %d limit 1", 
-                dbesc($arr['mid']), 
-                intval(local_channel())
-        );
-        if ($z && $i) {
-            $remote_id = $z[0]['id'];
-            $arr['id'] = $i[0]['id'];
-            // don't update if it has the same timestamp as the original
-            if ($arr['edited'] > $i[0]['edited'])
-                $x = item_store_update($arr, $execflag);
-        } else {
-            if (($i) && (intval($i[0]['item_deleted']))) {
-                // was partially deleted already, finish it off
-                q("delete from item where mid = '%s' and uid = %d", dbesc($arr['mid']), intval(local_channel())
-                );
-            }
-            logger('hubsites plugin: item_store= ' . json_encode($arr,true));
-            $x = item_store($arr, $execflag);
-        }
-        if ($x['success']) {
-            $item_id = $x['item_id'];
-            update_remote_id($channel, $item_id, $arr['item_type'], $pagetitle, $namespace, $remote_id, $arr['mid']);
-            $p['import_success'] = 1;
-        } else {
-            $p['import_success'] = 0;
-        }
-    }
-    return $pages;
-    
-}
-
-private function import_layouts($channel, $layouts) {
-    foreach ($layouts as &$p) {
-        
-        $arr = array();
-        $arr['item_type'] = ITEM_TYPE_PDL;
-        $namespace = 'PDL';
-        $arr['uid'] = $channel['channel_id'];
-        $arr['aid'] = $channel['channel_account_id'];
-        $pagetitle = $p['name'];
-        // See if the layout already exists
-        $iid = q("select iid from iconfig where k = 'PDL' and v = '%s' and cat = 'system'",
-                dbesc($pagetitle)
-        );
-        if($iid) {
-            // Get the existing layout info
-            $info = q("select mid,layout_mid,created,edited from item where id = %d",
-                    intval($iid[0]['iid'])
-            );
-            $arr['mid'] = $arr['parent_mid'] = $info[0]['mid'];
-            $arr['created'] = $info[0]['created'];
-            $arr['edited'] = (($p['edited']) ? datetime_convert('UTC', 'UTC', $p['edited']) : datetime_convert());
-        } else {
-            $arr['created'] = (($p['created']) ? datetime_convert('UTC', 'UTC', $p['created']) : datetime_convert());
-            $arr['edited'] = datetime_convert('UTC', 'UTC', '0000-00-00 00:00:00');
-            $arr['mid'] = $arr['parent_mid'] = item_message_id();
-        }
-        $arr['title'] = $p['description'];
-        $arr['body'] = file_get_contents($p['path']);
-        $arr['term'] = $p['term'];  // Not sure what this is supposed to be
-        
-        $arr['owner_xchan'] = get_observer_hash();
-        $arr['author_xchan'] = (($p['author_xchan']) ? $p['author_xchan'] : get_observer_hash());
-        $arr['mimetype'] = 'text/bbcode';
-
-        $remote_id = 0;
-
-        $z = q("select * from iconfig where v = '%s' and k = '%s' and cat = 'system' limit 1",
-                dbesc($pagetitle),
-                dbesc($namespace)
-        );
-
-        $i = q("select id, edited, item_deleted from item where mid = '%s' and uid = %d limit 1", 
-                dbesc($arr['mid']), 
-                intval(local_channel())
-        );
-        if ($z && $i) {
-            $remote_id = $z[0]['id'];
-            $arr['id'] = $i[0]['id'];
-            // don't update if it has the same timestamp as the original
-            if ($arr['edited'] > $i[0]['edited'])
-                $x = item_store_update($arr, $execflag);
-        } else {
-            if (($i) && (intval($i[0]['item_deleted']))) {
-                // was partially deleted already, finish it off
-                q("delete from item where mid = '%s' and uid = %d", dbesc($arr['mid']), intval(local_channel())
-                );
-            }
-            $x = item_store($arr, $execflag);
-        }
-        if ($x['success']) {
-            $item_id = $x['item_id'];
-            update_remote_id($channel, $item_id, $arr['item_type'], $pagetitle, $namespace, $remote_id, $arr['mid']);
-            $p['import_success'] = 1;
-        } else {
-            $p['import_success'] = 0;
-        }
-    }
-    return $layouts;
-    
-}
-
-
 }