diff options
| -rw-r--r-- | Zotlabs/Lib/Hashpath.php | 55 | ||||
| -rw-r--r-- | Zotlabs/Module/Photo.php | 6 | ||||
| -rw-r--r-- | Zotlabs/Photo/PhotoDriver.php | 31 | ||||
| -rw-r--r-- | doc/hidden_configs.bb | 2 | ||||
| -rw-r--r-- | include/import.php | 2 | ||||
| -rw-r--r-- | include/photo/photo_driver.php | 43 | ||||
| -rwxr-xr-x | util/storageconv | 113 |
7 files changed, 168 insertions, 84 deletions
diff --git a/Zotlabs/Lib/Hashpath.php b/Zotlabs/Lib/Hashpath.php new file mode 100644 index 000000000..f3b25d2b6 --- /dev/null +++ b/Zotlabs/Lib/Hashpath.php @@ -0,0 +1,55 @@ +<?php +namespace Zotlabs\Lib; + +/* + * Zotlabs\Lib\Hashpath + * + * Creates hashed directory structures for fast access and resistance to overloading any single directory with files. + * + * Takes a $hash which could be any string + * a $prefix which is where to place the hash directory in the filesystem, default is current directory + * use an empty string for $prefix to place hash directories directly off the root directory + * an optional $depth and $slice (default is 2) to indicate the hash level + * $depth = 1, 256 directories, suitable for < 384K records/files + * $depth = 2, 65536 directories, suitable for < 98M records/files + * $depth = 3, 16777216 directories, suitable for < 2.5B records/files + * ... + * The total number of records anticipated divided by the number of hash directories should generally be kept to + * less than 1500 entries for optimum performance though this varies by operating system and filesystem type. + * ext4 uses 32 bit inode numbers (~4B record limit) so use caution or alternative filesystem types with $depth above 3. + * an optional $mkdir (boolean) to recursively create the directory (ignoring errors) before returning + * + * examples: for a $hash of 'abcdefg' and prefix of 'path' the following paths are returned for $depth = 1 and $depth = 3 + * path/7d/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a + * path/7d/1a/54/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a + * + * see also: boot.php:os_mkdir() - here we provide the equivalent of mkdir -p with permissions of 770. + * + */ + +class Hashpath { + + static function path($hash, $prefix = '.', $depth = 1, $slice = 2, $mkdir = true, $alg = false) { + + if ($alg) + $hash = hash($alg, $hash); + + $start = 0; + if ($depth < 1) + $depth = 1; + $sluglen = $depth * $slice; + + do { + $slug = substr($hash, $start, $slice); + $prefix .= '/' . $slug; + $start += $slice; + $sluglen -= $slice; + } + while ($sluglen); + + if ($mkdir) + os_mkdir($prefix, STORAGE_DEFAULT_PERMISSIONS, true); + + return $prefix . '/' . $hash; + } +} diff --git a/Zotlabs/Module/Photo.php b/Zotlabs/Module/Photo.php index 041bdf5a2..ee360dac5 100644 --- a/Zotlabs/Module/Photo.php +++ b/Zotlabs/Module/Photo.php @@ -194,9 +194,9 @@ class Photo extends \Zotlabs\Web\Controller { $mimetype = $e[0]['mimetype']; $modified = strtotime($e[0]['edited'] . 'Z'); - if(intval($e[0]['os_storage'])) { + if(intval($e[0]['os_storage'])) $streaming = $data; - } + if($e[0]['allow_cid'] != '' || $e[0]['allow_gid'] != '' || $e[0]['deny_gid'] != '' || $e[0]['deny_gid'] != '') $prvcachecontrol = 'no-store, no-cache, must-revalidate'; } @@ -282,7 +282,7 @@ class Photo extends \Zotlabs\Web\Controller { header("Content-Length: " . (isset($filesize) ? $filesize : strlen($data))); // If it's a file resource, stream it. - if($streaming && $channel) { + if($streaming) { if(strpos($streaming,'store') !== false) $istream = fopen($streaming,'rb'); else diff --git a/Zotlabs/Photo/PhotoDriver.php b/Zotlabs/Photo/PhotoDriver.php index 94d2c3436..4c4f26e32 100644 --- a/Zotlabs/Photo/PhotoDriver.php +++ b/Zotlabs/Photo/PhotoDriver.php @@ -2,6 +2,8 @@ namespace Zotlabs\Photo; +use Zotlabs\Lib\Hashpath; + /** * @brief Abstract photo driver class. * @@ -505,18 +507,25 @@ abstract class PhotoDriver { * @return boolean */ public function storeThumbnail($arr, $scale = 0) { - - // We only process thumbnails here - if($scale == 0) - return false; - - $arr['imgscale'] = $scale; - - if(boolval(get_config('system','filesystem_storage_thumbnails', 0))) { - $channel = channelx_by_n($arr['uid']); + + // We only process thumbnails here + if($scale == 0) + return false; + + $arr['imgscale'] = $scale; + + if(boolval(get_config('system','photo_storage_type', 1))) { + $arr['os_storage'] = 1; - $arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale; - if(! $this->saveImage($arr['os_syspath'])) + + if (array_key_exists('uid', $arr) && ! in_array($scale, [ PHOTO_RES_PROFILE_300, PHOTO_RES_PROFILE_80, PHOTO_RES_PROFILE_48 ])) { + $channel = channelx_by_n($arr['uid']); + $arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale; + } + else + $arr['os_syspath'] = Hashpath::path($arr['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $scale; + + if (! $this->saveImage($arr['os_syspath'])) return false; } else diff --git a/doc/hidden_configs.bb b/doc/hidden_configs.bb index 42c9e67b8..27ea415bd 100644 --- a/doc/hidden_configs.bb +++ b/doc/hidden_configs.bb @@ -62,7 +62,7 @@ Options are: [*= system.email_notify_icon_url ] URL of image (32x32) to display in email notifications (HTML bodies). [*= system.expire_delivery_reports ] Expiration in days for delivery reports - default 10 [*= system.expire_limit ] Don't expire any more than this number of posts per channel per expiration run to keep from exhausting memory. Default 5000. - [*= system.filesystem_storage_thumbnails ] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2 + [*= system.photo_storage_type] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2 [*= system.hidden_version_siteinfo ] If true, do not report the software version on siteinfo pages (system.hide_version also hides the version on these pages, this setting *only* hides the version on siteinfo pages). [*= system.hide_help ] Don't display help documentation link in nav bar [*= system.hide_in_statistics ] Tell the red statistics servers to completely hide this hub in hub lists. diff --git a/include/import.php b/include/import.php index b512e1f11..8ce582ede 100644 --- a/include/import.php +++ b/include/import.php @@ -1469,7 +1469,7 @@ function sync_files($channel, $files) { fclose($fp); // Override remote hub thumbnails storage settings - if(! boolval(get_config('system','filesystem_storage_thumbnails', 0))) { + if(! boolval(get_config('system','photo_storage_type', 1))) { $p['os_storage'] = 0; $p['content'] = file_get_contents($stored_image); @unlink($stored_image); diff --git a/include/photo/photo_driver.php b/include/photo/photo_driver.php index 1ce2fd6e3..057711f1e 100644 --- a/include/photo/photo_driver.php +++ b/include/photo/photo_driver.php @@ -205,7 +205,11 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) { if($thing) $hash = photo_new_resource(); else { - $r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = 4 LIMIT 1", dbesc($xchan), intval(PHOTO_XCHAN)); + $r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = %d LIMIT 1", + dbesc($xchan), + intval(PHOTO_XCHAN), + intval(PHOTO_RES_PROFILE_300) + ); if($r) { $hash = $r[0]['resource_id']; $modified = $r[0]['edited']; @@ -316,26 +320,20 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) { 'filename' => basename($photo), 'album' => $album, 'photo_usage' => $flags, - 'imgscale' => 4, 'edited' => $modified, 'description' => (array_key_exists('etag', $hdrs) ? $hdrs['etag'] : ''), 'expires' => gmdate('Y-m-d H:i:s', (isset($expires) ? $expires : time() + 86400)) ]; - $r = $img->save($p); - if($r === false) - $photo_failure = true; + $r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300); $img->scaleImage(80); - $p['imgscale'] = 5; - $r = $img->save($p); - if($r === false) - $photo_failure = true; + $r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80); $img->scaleImage(48); - $p['imgscale'] = 6; - $r = $img->save($p); - if($r === false) + $r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48); + + if($r1 === false || $r2 === false || $r3 === false) $photo_failure = true; $photo = z_root() . '/photo/' . $hash . '-4'; @@ -424,31 +422,26 @@ function import_channel_photo($photo, $type, $aid, $uid) { 'resource_id' => $hash, 'filename' => $filename, 'album' => t('Profile Photos'), - 'photo_usage' => PHOTO_PROFILE, - 'imgscale' => 4, + 'photo_usage' => PHOTO_PROFILE ]; // photo size $img->scaleImageSquare(300); - $r = $img->save($p); - if($r === false) - $photo_failure = true; + $r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300); // thumb size $img->scaleImage(80); - $p['imgscale'] = 5; - $r = $img->save($p); - if($r === false) - $photo_failure = true; + $r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80); // micro size $img->scaleImage(48); - $p['imgscale'] = 6; - $r = $img->save($p); - if($r === false) + $r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48); + + if($r1 === false || $r2 === false || $r3 === false) $photo_failure = true; - } else { + } + else { logger('Invalid image.'); $photo_failure = true; } diff --git a/util/storageconv b/util/storageconv index 992c906b8..52bb77fbb 100755 --- a/util/storageconv +++ b/util/storageconv @@ -18,6 +18,8 @@ require_once('include/cli_startup.php'); cli_startup(); +use Zotlabs\Lib\Hashpath; + if($argc == 1) { usage(); killme(); @@ -25,15 +27,20 @@ if($argc == 1) { if($argc == 2) { - $storage = (intval(get_config('system','filesystem_storage_thumbnails', 0)) > 0 ? 1 : 0); + $storage = (intval(get_config('system','photo_storage_type', 1)) > 0 ? 1 : 0); echo 'Current storage set to: ' . ($storage ? 'filesystem' : 'SQL database') . PHP_EOL; switch($argv[1]) { case 'stats': $x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0"); echo 'Local images: ' . $x[0]['qty'] . PHP_EOL; - $x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND imgscale > 0"); - echo 'Thumbnails total: ' . $x[0]['qty'] . PHP_EOL; - $x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND os_storage != %d AND imgscale > 0", + $x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND imgscale > 0"); + echo 'Image thumbnails: ' . $x[0]['qty'] . PHP_EOL; + $xx = intval($x[0]['qty']); + $x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage IN (1, 2)"); + echo 'Imported profiles thumbnails: ' . $x[0]['qty'] . PHP_EOL; + $xx += intval($x[0]['qty']); + echo 'Thumbnails total: ' . $xx . PHP_EOL; + $x = q("SELECT COUNT(id) AS qty FROM photo WHERE os_storage != %d AND imgscale > 0", $storage ); echo 'Thumbnails to convert: ' . $x[0]['qty'] . PHP_EOL; @@ -41,87 +48,108 @@ if($argc == 2) { case 'fs': if($storage == 0) { - echo 'Please set system.filesystem_storage_thumbnails to 1 before move thumbnails to filesystem storage' . PHP_EOL; + echo 'Please set system.photo_storage_type to 1 before move thumbnails to filesystem storage' . PHP_EOL; break; } - $x = q("SELECT resource_id, content FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0"); + $cur_id = 0; + $i = 0; - if($x) { - foreach($x as $xx) { - - $n = q("SELECT id, imgscale, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0", - dbesc($xx['resource_id']), - $storage - ); + $r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 0 AND imgscale > 0"); + $max_num = $r[0]['max_num']; - $img_path = dbunescbin($xx['content']); - - foreach($n as $nn) { - - echo '.'; - - $filename = $img_path . '-' . $nn['imgscale']; + while ($i < $max_num) { + + $x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 0 AND imgscale > 0 ORDER BY id LIMIT 10", + intval($cur_id) + ); - if(! file_put_contents($filename, dbunescbin($nn['content']))) { - echo 'Failed to save file ' . $filename . PHP_EOL; + if($x) { + foreach($x as $xx) { + + if ($xx['uid'] == 0 || in_array($xx['imgscale'], [4, 5, 6])) + $filename = Hashpath::path($xx['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $xx['imgscale']; + else { + $z = q("SELECT channel_address FROM channel WHERE channel_id = %d", + intval($xx['uid']) + ); + $filename = 'store/' . $z[0]['channel_address'] . '/' . $xx['resource_id'] . '-' . $xx['imgscale']; + } + + if(! file_put_contents($filename, dbunescbin($xx['content']))) { + echo PHP_EOL . 'Failed to save file ' . $filename . PHP_EOL; continue; } - + $z = q("UPDATE photo SET content = '%s', os_storage = 1 WHERE id = %d", dbescbin($filename), - intval($nn['id']) + intval($xx['id']) ); if(! $z) { @unlink($filename); - echo 'Failed to update metadata for saved file ' . $filename . PHP_EOL; + echo PHP_EOL . 'Failed to update metadata for saved file ' . $filename . PHP_EOL; } + $cur_id = $xx['id']; + + echo '.'; + $i++; } } + echo ($i % 100 == 0 ? $i : ''); } + echo $i . PHP_EOL . 'Total thumbnails processed: ' . $i; break; case 'db': if($storage == 1) { - echo 'Please set system.filesystem_storage_thumbnails to 0 before move thumbnails to SQL database storage' . PHP_EOL; + echo 'Please set system.photo_storage_type to 0 before move thumbnails to SQL database storage' . PHP_EOL; break; } - $x = q("SELECT resource_id FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0"); - - if($x) { - foreach($x as $xx) { + $cur_id = 0; + $i = 0; - $n = q("SELECT id, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0", - dbesc($xx['resource_id']), - $storage - ); - - foreach($n as $nn) { - - echo '.'; + $r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 1 AND imgscale > 0"); + $max_num = $r[0]['max_num']; + + while ($i < $max_num) { + + $x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 1 AND imgscale > 0 ORDER BY id LIMIT 10", + intval($cur_id) + ); + + if($x) { + foreach($x as $xx) { - $filename = dbunescbin($nn['content']); + $filename = dbunescbin($xx['content']); + $content = file_get_contents($filename); if($content) { $z = q("UPDATE photo SET content = '%s', os_storage = 0 WHERE id = %d", dbescbin($content), - intval($nn['id']) + intval($xx['id']) ); if(! $z) { - echo 'Failed to update stored file metadata ' . $filename . PHP_EOL; + echo PHP_EOL . 'Failed to update stored file metadata ' . $filename . PHP_EOL; continue; } @unlink($filename); } else - echo 'Can not read file contents ' . $filename . PHP_EOL; + echo PHP_EOL . 'Can not read file contents ' . $filename . PHP_EOL; + + $cur_id = $xx['id']; + + echo '.'; + $i++; } } + echo ($i % 100 == 0 ? $i : ''); } + echo $i . PHP_EOL . 'Total files processed: ' . $i; break; default: @@ -129,6 +157,5 @@ if($argc == 2) { return; } - echo PHP_EOL; } |