Merge branch 'dev' into 'dev'

Add support filesystem storage for xchan profile photos

See merge request hubzilla/core!1898

7 files changed, 168 insertions, 84 deletions

diff --git a/Zotlabs/Lib/Hashpath.php b/Zotlabs/Lib/Hashpath.php
new file mode 100644
index 000000000..f3b25d2b6
--- /dev/null
+++ b/Zotlabs/Lib/Hashpath.php
@@ -0,0 +1,55 @@
+<?php
+namespace Zotlabs\Lib;
+
+/*
+ * Zotlabs\Lib\Hashpath
+ *
+ * Creates hashed directory structures for fast access and resistance to overloading any single directory with files.
+ *
+ * Takes a $hash which could be any string
+ * a $prefix which is where to place the hash directory in the filesystem, default is current directory
+ * use an empty string for $prefix to place hash directories directly off the root directory
+ * an optional $depth and $slice (default is 2) to indicate the hash level
+ * $depth = 1, 256 directories, suitable for < 384K records/files
+ * $depth = 2, 65536 directories, suitable for < 98M records/files
+ * $depth = 3, 16777216 directories, suitable for < 2.5B records/files
+ * ...
+ * The total number of records anticipated divided by the number of hash directories should generally be kept to
+ * less than 1500 entries for optimum performance though this varies by operating system and filesystem type.
+ * ext4 uses 32 bit inode numbers (~4B record limit) so use caution or alternative filesystem types with $depth above 3. 
+ * an optional $mkdir (boolean) to recursively create the directory (ignoring errors) before returning
+ *
+ * examples: for a $hash of 'abcdefg' and prefix of 'path' the following paths are returned for $depth = 1 and $depth = 3
+ *    path/7d/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a
+ *    path/7d/1a/54/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a
+ *
+ * see also: boot.php:os_mkdir() - here we provide the equivalent of mkdir -p with permissions of 770. 
+ *
+ */
+ 
+class Hashpath {
+
+	static function path($hash, $prefix = '.', $depth = 1, $slice = 2, $mkdir = true, $alg = false) {
+
+		if ($alg)
+			$hash = hash($alg, $hash);
+
+		$start = 0;
+		if ($depth < 1)
+			$depth = 1;
+		$sluglen = $depth * $slice;
+
+		do {
+			$slug	  = substr($hash, $start, $slice);
+			$prefix  .= '/' . $slug;
+			$start	 += $slice;
+			$sluglen -= $slice;
+		}
+		while ($sluglen);
+
+		if ($mkdir)
+			os_mkdir($prefix, STORAGE_DEFAULT_PERMISSIONS, true);
+		
+		return $prefix . '/' . $hash;
+	}
+}
diff --git a/Zotlabs/Module/Photo.php b/Zotlabs/Module/Photo.php
index 041bdf5a2..ee360dac5 100644
--- a/Zotlabs/Module/Photo.php
+++ b/Zotlabs/Module/Photo.php
@@ -194,9 +194,9 @@ class Photo extends \Zotlabs\Web\Controller {
 					$mimetype = $e[0]['mimetype'];
 					$modified = strtotime($e[0]['edited'] . 'Z');
 
-					if(intval($e[0]['os_storage'])) {
+					if(intval($e[0]['os_storage']))
 						$streaming = $data;
-					}
+
 					if($e[0]['allow_cid'] != '' || $e[0]['allow_gid'] != '' || $e[0]['deny_gid'] != '' || $e[0]['deny_gid'] != '')
 						$prvcachecontrol = 'no-store, no-cache, must-revalidate';
 				}
@@ -282,7 +282,7 @@ class Photo extends \Zotlabs\Web\Controller {
 		header("Content-Length: " . (isset($filesize) ? $filesize : strlen($data)));
 
 		// If it's a file resource, stream it. 
-		if($streaming && $channel) {
+		if($streaming) {
 			if(strpos($streaming,'store') !== false)
 				$istream = fopen($streaming,'rb');
 			else
diff --git a/Zotlabs/Photo/PhotoDriver.php b/Zotlabs/Photo/PhotoDriver.php
index 94d2c3436..4c4f26e32 100644
--- a/Zotlabs/Photo/PhotoDriver.php
+++ b/Zotlabs/Photo/PhotoDriver.php
@@ -2,6 +2,8 @@
 
 namespace Zotlabs\Photo;
 
+use Zotlabs\Lib\Hashpath; 
+
 /**
  * @brief Abstract photo driver class.
  *
@@ -505,18 +507,25 @@ abstract class PhotoDriver {
 	 * @return boolean
 	 */
 	public function storeThumbnail($arr, $scale = 0) {
-	    
-	    // We only process thumbnails here
-	    if($scale == 0)
-	        return false;
-	    
-	    $arr['imgscale'] = $scale;
-
-		if(boolval(get_config('system','filesystem_storage_thumbnails', 0))) {
-			$channel = channelx_by_n($arr['uid']);
+
+		// We only process thumbnails here
+		if($scale == 0)
+			return false;
+
+		$arr['imgscale'] = $scale;
+
+		if(boolval(get_config('system','photo_storage_type', 1))) {
+
 			$arr['os_storage'] = 1;
-			$arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale;
-			if(! $this->saveImage($arr['os_syspath']))
+
+			if (array_key_exists('uid', $arr) && ! in_array($scale, [ PHOTO_RES_PROFILE_300, PHOTO_RES_PROFILE_80, PHOTO_RES_PROFILE_48 ])) {
+				$channel = channelx_by_n($arr['uid']);
+				$arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale;
+			}
+			else
+				$arr['os_syspath'] = Hashpath::path($arr['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $scale;
+
+			if (! $this->saveImage($arr['os_syspath']))
 				return false;
 		}
 		else
diff --git a/doc/hidden_configs.bb b/doc/hidden_configs.bb
index 42c9e67b8..27ea415bd 100644
--- a/doc/hidden_configs.bb
+++ b/doc/hidden_configs.bb
@@ -62,7 +62,7 @@ Options are:
   [*= system.email_notify_icon_url ] URL of image (32x32) to display in email notifications (HTML bodies).
   [*= system.expire_delivery_reports ] Expiration in days for delivery reports - default 10
   [*= system.expire_limit ] Don't expire any more than this number of posts per channel per expiration run to keep from exhausting memory. Default 5000.
-  [*= system.filesystem_storage_thumbnails ] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2
+  [*= system.photo_storage_type] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2
   [*= system.hidden_version_siteinfo ] If true, do not report the software version on siteinfo pages (system.hide_version also hides the version on these pages, this setting *only* hides the version on siteinfo pages).
   [*= system.hide_help ] Don't display help documentation link in nav bar
   [*= system.hide_in_statistics ] Tell the red statistics servers to completely hide this hub in hub lists.
diff --git a/include/import.php b/include/import.php
index b512e1f11..8ce582ede 100644
--- a/include/import.php
+++ b/include/import.php
@@ -1469,7 +1469,7 @@ function sync_files($channel, $files) {
 						fclose($fp);
 
 						// Override remote hub thumbnails storage settings
-						if(! boolval(get_config('system','filesystem_storage_thumbnails', 0))) {
+						if(! boolval(get_config('system','photo_storage_type', 1))) {
 							$p['os_storage'] = 0;
 							$p['content'] = file_get_contents($stored_image);
 							@unlink($stored_image);
diff --git a/include/photo/photo_driver.php b/include/photo/photo_driver.php
index 1ce2fd6e3..057711f1e 100644
--- a/include/photo/photo_driver.php
+++ b/include/photo/photo_driver.php
@@ -205,7 +205,11 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) {
 	if($thing)
 		$hash = photo_new_resource();
 	else {
-		$r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = 4 LIMIT 1", dbesc($xchan), intval(PHOTO_XCHAN));
+		$r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = %d LIMIT 1",
+			dbesc($xchan),
+			intval(PHOTO_XCHAN),
+			intval(PHOTO_RES_PROFILE_300)
+		);
 		if($r) {
 			$hash = $r[0]['resource_id'];
 			$modified = $r[0]['edited'];
@@ -316,26 +320,20 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) {
 				'filename' => basename($photo),
 				'album' => $album,
 				'photo_usage' => $flags,
-				'imgscale' => 4,
 				'edited' => $modified,
 				'description' => (array_key_exists('etag', $hdrs) ? $hdrs['etag'] : ''),
 				'expires' => gmdate('Y-m-d H:i:s', (isset($expires) ? $expires : time() + 86400))
 			];
 
-			$r = $img->save($p);
-			if($r === false)
-				$photo_failure = true;
+			$r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300);
 
 			$img->scaleImage(80);
-			$p['imgscale'] = 5;
-			$r = $img->save($p);
-			if($r === false)
-				$photo_failure = true;
+			$r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80);
 
 			$img->scaleImage(48);
-			$p['imgscale'] = 6;
-			$r = $img->save($p);
-			if($r === false)
+			$r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48);
+
+			if($r1 === false || $r2 === false || $r3 === false)
 				$photo_failure = true;
 
 			$photo = z_root() . '/photo/' . $hash . '-4';
@@ -424,31 +422,26 @@ function import_channel_photo($photo, $type, $aid, $uid) {
 				'resource_id' => $hash,
 				'filename' => $filename,
 				'album' => t('Profile Photos'),
-				'photo_usage' => PHOTO_PROFILE,
-				'imgscale' => 4,
+				'photo_usage' => PHOTO_PROFILE
 		];
 
 		// photo size
 		$img->scaleImageSquare(300);
-		$r = $img->save($p);
-		if($r === false)
-			$photo_failure = true;
+		$r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300);
 
 		// thumb size
 		$img->scaleImage(80);
-		$p['imgscale'] = 5;
-		$r = $img->save($p);
-		if($r === false)
-			$photo_failure = true;
+		$r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80);
 
 		// micro size
 		$img->scaleImage(48);
-		$p['imgscale'] = 6;
-		$r = $img->save($p);
-		if($r === false)
+		$r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48);
+
+		if($r1 === false || $r2 === false || $r3 === false)
 			$photo_failure = true;
 
-	} else {
+	}
+	else {
 		logger('Invalid image.');
 		$photo_failure = true;
 	}
diff --git a/util/storageconv b/util/storageconv
index 992c906b8..52bb77fbb 100755
--- a/util/storageconv
+++ b/util/storageconv
@@ -18,6 +18,8 @@ require_once('include/cli_startup.php');
 
 cli_startup();
 
+use Zotlabs\Lib\Hashpath;
+
 if($argc == 1) {
 	usage();
 	killme();
@@ -25,15 +27,20 @@ if($argc == 1) {
 
 if($argc == 2) {
 	
-	$storage = (intval(get_config('system','filesystem_storage_thumbnails', 0)) > 0 ? 1 : 0);
+	$storage = (intval(get_config('system','photo_storage_type', 1)) > 0 ? 1 : 0);
 	echo 'Current storage set to: ' . ($storage ? 'filesystem' : 'SQL database') . PHP_EOL;
 	switch($argv[1]) {
 		case 'stats':
 			$x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
 			echo 'Local images: ' . $x[0]['qty'] . PHP_EOL;
-			$x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND imgscale > 0");
-			echo 'Thumbnails total: ' . $x[0]['qty'] . PHP_EOL;
-			$x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND os_storage != %d AND imgscale > 0",
+			$x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND imgscale > 0");
+			echo 'Image thumbnails: ' . $x[0]['qty'] . PHP_EOL;
+			$xx = intval($x[0]['qty']);
+			$x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage IN (1, 2)");
+			echo 'Imported profiles thumbnails: ' . $x[0]['qty'] . PHP_EOL;
+			$xx += intval($x[0]['qty']);			
+			echo 'Thumbnails total: ' . $xx . PHP_EOL;
+			$x = q("SELECT COUNT(id) AS qty FROM photo WHERE os_storage != %d AND imgscale > 0",
 				$storage
 			);
 			echo 'Thumbnails to convert: ' . $x[0]['qty'] . PHP_EOL;
@@ -41,87 +48,108 @@ if($argc == 2) {
 			
 		case 'fs':
 			if($storage == 0) {
-				echo 'Please set system.filesystem_storage_thumbnails to 1 before move thumbnails to filesystem storage' . PHP_EOL;
+				echo 'Please set system.photo_storage_type to 1 before move thumbnails to filesystem storage' . PHP_EOL;
 				break;
 			}
 
-			$x = q("SELECT resource_id, content FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
+			$cur_id = 0;
+			$i = 0;
 
-			if($x) {
-				foreach($x as $xx) {
-					
-					$n = q("SELECT id, imgscale, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0",
-						dbesc($xx['resource_id']),
-						$storage
-					);
+			$r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 0 AND imgscale > 0");
+			$max_num = $r[0]['max_num'];
 
-					$img_path = dbunescbin($xx['content']);
-					
-					foreach($n as $nn) {
-						
-						echo '.';
-						
-						$filename = $img_path . '-' . $nn['imgscale'];
+			while ($i < $max_num) {
+
+				$x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 0 AND imgscale > 0 ORDER BY id LIMIT 10",
+					intval($cur_id)
+				);
 
-						if(! file_put_contents($filename, dbunescbin($nn['content']))) {
-							echo 'Failed to save file ' . $filename . PHP_EOL;
+				if($x) {
+					foreach($x as $xx) {
+
+						if ($xx['uid'] == 0 || in_array($xx['imgscale'], [4, 5, 6]))
+							$filename = Hashpath::path($xx['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $xx['imgscale'];
+						else {
+							$z = q("SELECT channel_address FROM channel WHERE channel_id = %d",
+								intval($xx['uid'])
+							);
+							$filename = 'store/' . $z[0]['channel_address'] . '/' . $xx['resource_id'] . '-' . $xx['imgscale'];
+						}
+
+						if(! file_put_contents($filename, dbunescbin($xx['content']))) {
+							echo PHP_EOL . 'Failed to save file ' . $filename . PHP_EOL;
 							continue;
 						}
-					
+				
 						$z = q("UPDATE photo SET content = '%s', os_storage = 1 WHERE id = %d",
 							dbescbin($filename),
-							intval($nn['id'])
+							intval($xx['id'])
 						);
 						if(! $z) {
 							@unlink($filename);
-							echo 'Failed to update metadata for saved file ' . $filename . PHP_EOL;
+							echo PHP_EOL . 'Failed to update metadata for saved file ' . $filename . PHP_EOL;
 						}
 
+						$cur_id = $xx['id'];
+
+						echo '.';
+						$i++;
 					}
 				}
+				echo ($i % 100 == 0 ? $i : '');
 			}
+			echo $i . PHP_EOL . 'Total thumbnails processed: ' . $i;
 			break;
 			
 		case 'db':
 			if($storage == 1) {
-				echo 'Please set system.filesystem_storage_thumbnails to 0 before move thumbnails to SQL database storage' . PHP_EOL;
+				echo 'Please set system.photo_storage_type to 0 before move thumbnails to SQL database storage' . PHP_EOL;
 				break;
 			}
 
-			$x = q("SELECT resource_id FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
-			
-			if($x) {
-				foreach($x as $xx) {
+			$cur_id = 0;
+			$i = 0;
 
-					$n = q("SELECT id, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0",
-						dbesc($xx['resource_id']),
-						$storage
-					);
-				
-					foreach($n as $nn) {
-						
-						echo '.';
+			$r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 1 AND imgscale > 0");
+			$max_num = $r[0]['max_num'];
+
+                        while ($i < $max_num) {
+
+                                $x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 1 AND imgscale > 0 ORDER BY id LIMIT 10",
+                                        intval($cur_id)
+                                );
+			
+				if($x) {
+					foreach($x as $xx) {
 						
-						$filename = dbunescbin($nn['content']);
+						$filename = dbunescbin($xx['content']);
+
 						$content = file_get_contents($filename);
 						if($content) {
 						
 							$z = q("UPDATE photo SET content = '%s', os_storage = 0 WHERE id = %d",
 								dbescbin($content),
-								intval($nn['id'])
+								intval($xx['id'])
 							);
 							if(! $z) {
-								echo 'Failed to update stored file metadata ' . $filename . PHP_EOL;
+								echo PHP_EOL . 'Failed to update stored file metadata ' . $filename . PHP_EOL;
 								continue;
 							}
 					
 							@unlink($filename);
 						}
 						else
-							echo 'Can not read file contents ' . $filename . PHP_EOL;
+							echo PHP_EOL . 'Can not read file contents ' . $filename . PHP_EOL;
+
+						$cur_id = $xx['id'];
+
+						echo '.';
+						$i++;
 					}
 				}
+				echo ($i % 100 == 0 ? $i : '');
 			}
+			echo $i . PHP_EOL . 'Total files processed: ' . $i;
 			break;
 			
 		default:
@@ -129,6 +157,5 @@ if($argc == 2) {
 			return;
 
 	}
-	
 	echo PHP_EOL;
 }