aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMario <mario@mariovavti.com>2021-01-24 20:03:35 +0000
committerMario <mario@mariovavti.com>2021-01-24 20:03:35 +0000
commit77793e17c04b1f67c10a83e7877c36a8b11ddbaa (patch)
tree293729c08f5f0193a6445f641b5ac09d43168d3b
parente486442eb1fdda0c4efa7eafc6d876b3c2c4b0ea (diff)
parent552796286e4a2a61cc6020b7aa785dc88553e2d0 (diff)
downloadvolse-hubzilla-77793e17c04b1f67c10a83e7877c36a8b11ddbaa.tar.gz
volse-hubzilla-77793e17c04b1f67c10a83e7877c36a8b11ddbaa.tar.bz2
volse-hubzilla-77793e17c04b1f67c10a83e7877c36a8b11ddbaa.zip
Merge branch 'dev' into 'dev'
Add support filesystem storage for xchan profile photos See merge request hubzilla/core!1898
-rw-r--r--Zotlabs/Lib/Hashpath.php55
-rw-r--r--Zotlabs/Module/Photo.php6
-rw-r--r--Zotlabs/Photo/PhotoDriver.php31
-rw-r--r--doc/hidden_configs.bb2
-rw-r--r--include/import.php2
-rw-r--r--include/photo/photo_driver.php43
-rwxr-xr-xutil/storageconv113
7 files changed, 168 insertions, 84 deletions
diff --git a/Zotlabs/Lib/Hashpath.php b/Zotlabs/Lib/Hashpath.php
new file mode 100644
index 000000000..f3b25d2b6
--- /dev/null
+++ b/Zotlabs/Lib/Hashpath.php
@@ -0,0 +1,55 @@
+<?php
+namespace Zotlabs\Lib;
+
+/*
+ * Zotlabs\Lib\Hashpath
+ *
+ * Creates hashed directory structures for fast access and resistance to overloading any single directory with files.
+ *
+ * Takes a $hash which could be any string
+ * a $prefix which is where to place the hash directory in the filesystem, default is current directory
+ * use an empty string for $prefix to place hash directories directly off the root directory
+ * an optional $depth and $slice (default is 2) to indicate the hash level
+ * $depth = 1, 256 directories, suitable for < 384K records/files
+ * $depth = 2, 65536 directories, suitable for < 98M records/files
+ * $depth = 3, 16777216 directories, suitable for < 2.5B records/files
+ * ...
+ * The total number of records anticipated divided by the number of hash directories should generally be kept to
+ * less than 1500 entries for optimum performance though this varies by operating system and filesystem type.
+ * ext4 uses 32 bit inode numbers (~4B record limit) so use caution or alternative filesystem types with $depth above 3.
+ * an optional $mkdir (boolean) to recursively create the directory (ignoring errors) before returning
+ *
+ * examples: for a $hash of 'abcdefg' and prefix of 'path' the following paths are returned for $depth = 1 and $depth = 3
+ * path/7d/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a
+ * path/7d/1a/54/7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a
+ *
+ * see also: boot.php:os_mkdir() - here we provide the equivalent of mkdir -p with permissions of 770.
+ *
+ */
+
+class Hashpath {
+
+ static function path($hash, $prefix = '.', $depth = 1, $slice = 2, $mkdir = true, $alg = false) {
+
+ if ($alg)
+ $hash = hash($alg, $hash);
+
+ $start = 0;
+ if ($depth < 1)
+ $depth = 1;
+ $sluglen = $depth * $slice;
+
+ do {
+ $slug = substr($hash, $start, $slice);
+ $prefix .= '/' . $slug;
+ $start += $slice;
+ $sluglen -= $slice;
+ }
+ while ($sluglen);
+
+ if ($mkdir)
+ os_mkdir($prefix, STORAGE_DEFAULT_PERMISSIONS, true);
+
+ return $prefix . '/' . $hash;
+ }
+}
diff --git a/Zotlabs/Module/Photo.php b/Zotlabs/Module/Photo.php
index 041bdf5a2..ee360dac5 100644
--- a/Zotlabs/Module/Photo.php
+++ b/Zotlabs/Module/Photo.php
@@ -194,9 +194,9 @@ class Photo extends \Zotlabs\Web\Controller {
$mimetype = $e[0]['mimetype'];
$modified = strtotime($e[0]['edited'] . 'Z');
- if(intval($e[0]['os_storage'])) {
+ if(intval($e[0]['os_storage']))
$streaming = $data;
- }
+
if($e[0]['allow_cid'] != '' || $e[0]['allow_gid'] != '' || $e[0]['deny_gid'] != '' || $e[0]['deny_gid'] != '')
$prvcachecontrol = 'no-store, no-cache, must-revalidate';
}
@@ -282,7 +282,7 @@ class Photo extends \Zotlabs\Web\Controller {
header("Content-Length: " . (isset($filesize) ? $filesize : strlen($data)));
// If it's a file resource, stream it.
- if($streaming && $channel) {
+ if($streaming) {
if(strpos($streaming,'store') !== false)
$istream = fopen($streaming,'rb');
else
diff --git a/Zotlabs/Photo/PhotoDriver.php b/Zotlabs/Photo/PhotoDriver.php
index 94d2c3436..4c4f26e32 100644
--- a/Zotlabs/Photo/PhotoDriver.php
+++ b/Zotlabs/Photo/PhotoDriver.php
@@ -2,6 +2,8 @@
namespace Zotlabs\Photo;
+use Zotlabs\Lib\Hashpath;
+
/**
* @brief Abstract photo driver class.
*
@@ -505,18 +507,25 @@ abstract class PhotoDriver {
* @return boolean
*/
public function storeThumbnail($arr, $scale = 0) {
-
- // We only process thumbnails here
- if($scale == 0)
- return false;
-
- $arr['imgscale'] = $scale;
-
- if(boolval(get_config('system','filesystem_storage_thumbnails', 0))) {
- $channel = channelx_by_n($arr['uid']);
+
+ // We only process thumbnails here
+ if($scale == 0)
+ return false;
+
+ $arr['imgscale'] = $scale;
+
+ if(boolval(get_config('system','photo_storage_type', 1))) {
+
$arr['os_storage'] = 1;
- $arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale;
- if(! $this->saveImage($arr['os_syspath']))
+
+ if (array_key_exists('uid', $arr) && ! in_array($scale, [ PHOTO_RES_PROFILE_300, PHOTO_RES_PROFILE_80, PHOTO_RES_PROFILE_48 ])) {
+ $channel = channelx_by_n($arr['uid']);
+ $arr['os_syspath'] = 'store/' . $channel['channel_address'] . '/' . $arr['os_path'] . '-' . $scale;
+ }
+ else
+ $arr['os_syspath'] = Hashpath::path($arr['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $scale;
+
+ if (! $this->saveImage($arr['os_syspath']))
return false;
}
else
diff --git a/doc/hidden_configs.bb b/doc/hidden_configs.bb
index 42c9e67b8..27ea415bd 100644
--- a/doc/hidden_configs.bb
+++ b/doc/hidden_configs.bb
@@ -62,7 +62,7 @@ Options are:
[*= system.email_notify_icon_url ] URL of image (32x32) to display in email notifications (HTML bodies).
[*= system.expire_delivery_reports ] Expiration in days for delivery reports - default 10
[*= system.expire_limit ] Don't expire any more than this number of posts per channel per expiration run to keep from exhausting memory. Default 5000.
- [*= system.filesystem_storage_thumbnails ] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2
+ [*= system.photo_storage_type] If '1', use filesystem instead SQL database to store thumbnails. Default is '0'. Introduced in 4.2
[*= system.hidden_version_siteinfo ] If true, do not report the software version on siteinfo pages (system.hide_version also hides the version on these pages, this setting *only* hides the version on siteinfo pages).
[*= system.hide_help ] Don't display help documentation link in nav bar
[*= system.hide_in_statistics ] Tell the red statistics servers to completely hide this hub in hub lists.
diff --git a/include/import.php b/include/import.php
index b512e1f11..8ce582ede 100644
--- a/include/import.php
+++ b/include/import.php
@@ -1469,7 +1469,7 @@ function sync_files($channel, $files) {
fclose($fp);
// Override remote hub thumbnails storage settings
- if(! boolval(get_config('system','filesystem_storage_thumbnails', 0))) {
+ if(! boolval(get_config('system','photo_storage_type', 1))) {
$p['os_storage'] = 0;
$p['content'] = file_get_contents($stored_image);
@unlink($stored_image);
diff --git a/include/photo/photo_driver.php b/include/photo/photo_driver.php
index 1ce2fd6e3..057711f1e 100644
--- a/include/photo/photo_driver.php
+++ b/include/photo/photo_driver.php
@@ -205,7 +205,11 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) {
if($thing)
$hash = photo_new_resource();
else {
- $r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = 4 LIMIT 1", dbesc($xchan), intval(PHOTO_XCHAN));
+ $r = q("SELECT resource_id, edited, mimetype, expires, description FROM photo WHERE xchan = '%s' AND photo_usage = %d AND imgscale = %d LIMIT 1",
+ dbesc($xchan),
+ intval(PHOTO_XCHAN),
+ intval(PHOTO_RES_PROFILE_300)
+ );
if($r) {
$hash = $r[0]['resource_id'];
$modified = $r[0]['edited'];
@@ -316,26 +320,20 @@ function import_xchan_photo($photo, $xchan, $thing = false, $force = false) {
'filename' => basename($photo),
'album' => $album,
'photo_usage' => $flags,
- 'imgscale' => 4,
'edited' => $modified,
'description' => (array_key_exists('etag', $hdrs) ? $hdrs['etag'] : ''),
'expires' => gmdate('Y-m-d H:i:s', (isset($expires) ? $expires : time() + 86400))
];
- $r = $img->save($p);
- if($r === false)
- $photo_failure = true;
+ $r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300);
$img->scaleImage(80);
- $p['imgscale'] = 5;
- $r = $img->save($p);
- if($r === false)
- $photo_failure = true;
+ $r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80);
$img->scaleImage(48);
- $p['imgscale'] = 6;
- $r = $img->save($p);
- if($r === false)
+ $r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48);
+
+ if($r1 === false || $r2 === false || $r3 === false)
$photo_failure = true;
$photo = z_root() . '/photo/' . $hash . '-4';
@@ -424,31 +422,26 @@ function import_channel_photo($photo, $type, $aid, $uid) {
'resource_id' => $hash,
'filename' => $filename,
'album' => t('Profile Photos'),
- 'photo_usage' => PHOTO_PROFILE,
- 'imgscale' => 4,
+ 'photo_usage' => PHOTO_PROFILE
];
// photo size
$img->scaleImageSquare(300);
- $r = $img->save($p);
- if($r === false)
- $photo_failure = true;
+ $r1 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_300);
// thumb size
$img->scaleImage(80);
- $p['imgscale'] = 5;
- $r = $img->save($p);
- if($r === false)
- $photo_failure = true;
+ $r2 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_80);
// micro size
$img->scaleImage(48);
- $p['imgscale'] = 6;
- $r = $img->save($p);
- if($r === false)
+ $r3 = $img->storeThumbnail($p, PHOTO_RES_PROFILE_48);
+
+ if($r1 === false || $r2 === false || $r3 === false)
$photo_failure = true;
- } else {
+ }
+ else {
logger('Invalid image.');
$photo_failure = true;
}
diff --git a/util/storageconv b/util/storageconv
index 992c906b8..52bb77fbb 100755
--- a/util/storageconv
+++ b/util/storageconv
@@ -18,6 +18,8 @@ require_once('include/cli_startup.php');
cli_startup();
+use Zotlabs\Lib\Hashpath;
+
if($argc == 1) {
usage();
killme();
@@ -25,15 +27,20 @@ if($argc == 1) {
if($argc == 2) {
- $storage = (intval(get_config('system','filesystem_storage_thumbnails', 0)) > 0 ? 1 : 0);
+ $storage = (intval(get_config('system','photo_storage_type', 1)) > 0 ? 1 : 0);
echo 'Current storage set to: ' . ($storage ? 'filesystem' : 'SQL database') . PHP_EOL;
switch($argv[1]) {
case 'stats':
$x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
echo 'Local images: ' . $x[0]['qty'] . PHP_EOL;
- $x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND imgscale > 0");
- echo 'Thumbnails total: ' . $x[0]['qty'] . PHP_EOL;
- $x = q("SELECT COUNT(id) AS qty FROM photo WHERE resource_id IN (SELECT DISTINCT resource_id FROM photo WHERE photo_usage = 0 and os_storage = 1) AND os_storage != %d AND imgscale > 0",
+ $x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage = 0 AND imgscale > 0");
+ echo 'Image thumbnails: ' . $x[0]['qty'] . PHP_EOL;
+ $xx = intval($x[0]['qty']);
+ $x = q("SELECT COUNT(resource_id) AS qty FROM photo WHERE photo_usage IN (1, 2)");
+ echo 'Imported profiles thumbnails: ' . $x[0]['qty'] . PHP_EOL;
+ $xx += intval($x[0]['qty']);
+ echo 'Thumbnails total: ' . $xx . PHP_EOL;
+ $x = q("SELECT COUNT(id) AS qty FROM photo WHERE os_storage != %d AND imgscale > 0",
$storage
);
echo 'Thumbnails to convert: ' . $x[0]['qty'] . PHP_EOL;
@@ -41,87 +48,108 @@ if($argc == 2) {
case 'fs':
if($storage == 0) {
- echo 'Please set system.filesystem_storage_thumbnails to 1 before move thumbnails to filesystem storage' . PHP_EOL;
+ echo 'Please set system.photo_storage_type to 1 before move thumbnails to filesystem storage' . PHP_EOL;
break;
}
- $x = q("SELECT resource_id, content FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
+ $cur_id = 0;
+ $i = 0;
- if($x) {
- foreach($x as $xx) {
-
- $n = q("SELECT id, imgscale, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0",
- dbesc($xx['resource_id']),
- $storage
- );
+ $r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 0 AND imgscale > 0");
+ $max_num = $r[0]['max_num'];
- $img_path = dbunescbin($xx['content']);
-
- foreach($n as $nn) {
-
- echo '.';
-
- $filename = $img_path . '-' . $nn['imgscale'];
+ while ($i < $max_num) {
+
+ $x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 0 AND imgscale > 0 ORDER BY id LIMIT 10",
+ intval($cur_id)
+ );
- if(! file_put_contents($filename, dbunescbin($nn['content']))) {
- echo 'Failed to save file ' . $filename . PHP_EOL;
+ if($x) {
+ foreach($x as $xx) {
+
+ if ($xx['uid'] == 0 || in_array($xx['imgscale'], [4, 5, 6]))
+ $filename = Hashpath::path($xx['resource_id'], 'store/[data]/[xchan]', 2, 1) . '-' . $xx['imgscale'];
+ else {
+ $z = q("SELECT channel_address FROM channel WHERE channel_id = %d",
+ intval($xx['uid'])
+ );
+ $filename = 'store/' . $z[0]['channel_address'] . '/' . $xx['resource_id'] . '-' . $xx['imgscale'];
+ }
+
+ if(! file_put_contents($filename, dbunescbin($xx['content']))) {
+ echo PHP_EOL . 'Failed to save file ' . $filename . PHP_EOL;
continue;
}
-
+
$z = q("UPDATE photo SET content = '%s', os_storage = 1 WHERE id = %d",
dbescbin($filename),
- intval($nn['id'])
+ intval($xx['id'])
);
if(! $z) {
@unlink($filename);
- echo 'Failed to update metadata for saved file ' . $filename . PHP_EOL;
+ echo PHP_EOL . 'Failed to update metadata for saved file ' . $filename . PHP_EOL;
}
+ $cur_id = $xx['id'];
+
+ echo '.';
+ $i++;
}
}
+ echo ($i % 100 == 0 ? $i : '');
}
+ echo $i . PHP_EOL . 'Total thumbnails processed: ' . $i;
break;
case 'db':
if($storage == 1) {
- echo 'Please set system.filesystem_storage_thumbnails to 0 before move thumbnails to SQL database storage' . PHP_EOL;
+ echo 'Please set system.photo_storage_type to 0 before move thumbnails to SQL database storage' . PHP_EOL;
break;
}
- $x = q("SELECT resource_id FROM photo WHERE photo_usage = 0 AND os_storage = 1 AND imgscale = 0");
-
- if($x) {
- foreach($x as $xx) {
+ $cur_id = 0;
+ $i = 0;
- $n = q("SELECT id, content FROM photo WHERE resource_id = '%s' AND os_storage != %d AND imgscale > 0",
- dbesc($xx['resource_id']),
- $storage
- );
-
- foreach($n as $nn) {
-
- echo '.';
+ $r = dbq("SELECT COUNT(id) AS max_num FROM photo WHERE os_storage = 1 AND imgscale > 0");
+ $max_num = $r[0]['max_num'];
+
+ while ($i < $max_num) {
+
+ $x = q("SELECT id, uid, resource_id, content, imgscale FROM photo WHERE id > %d AND os_storage = 1 AND imgscale > 0 ORDER BY id LIMIT 10",
+ intval($cur_id)
+ );
+
+ if($x) {
+ foreach($x as $xx) {
- $filename = dbunescbin($nn['content']);
+ $filename = dbunescbin($xx['content']);
+
$content = file_get_contents($filename);
if($content) {
$z = q("UPDATE photo SET content = '%s', os_storage = 0 WHERE id = %d",
dbescbin($content),
- intval($nn['id'])
+ intval($xx['id'])
);
if(! $z) {
- echo 'Failed to update stored file metadata ' . $filename . PHP_EOL;
+ echo PHP_EOL . 'Failed to update stored file metadata ' . $filename . PHP_EOL;
continue;
}
@unlink($filename);
}
else
- echo 'Can not read file contents ' . $filename . PHP_EOL;
+ echo PHP_EOL . 'Can not read file contents ' . $filename . PHP_EOL;
+
+ $cur_id = $xx['id'];
+
+ echo '.';
+ $i++;
}
}
+ echo ($i % 100 == 0 ? $i : '');
}
+ echo $i . PHP_EOL . 'Total files processed: ' . $i;
break;
default:
@@ -129,6 +157,5 @@ if($argc == 2) {
return;
}
-
echo PHP_EOL;
}