diff options
| -rw-r--r-- | Zotlabs/Daemon/Cron.php | 68 | ||||
| -rw-r--r-- | Zotlabs/Daemon/Notifier.php | 11 | ||||
| -rw-r--r-- | Zotlabs/Lib/Activity.php | 66 | ||||
| -rw-r--r-- | Zotlabs/Lib/Config.php | 4 | ||||
| -rw-r--r-- | Zotlabs/Lib/IConfig.php | 32 | ||||
| -rw-r--r-- | Zotlabs/Lib/Libzot.php | 8 | ||||
| -rw-r--r-- | Zotlabs/Lib/ObjCache.php | 40 | ||||
| -rw-r--r-- | Zotlabs/Module/Viewsrc.php | 39 | ||||
| -rw-r--r-- | include/items.php | 10 | ||||
| -rw-r--r-- | include/text.php | 2 |
10 files changed, 207 insertions, 73 deletions
diff --git a/Zotlabs/Daemon/Cron.php b/Zotlabs/Daemon/Cron.php index 131abe2e1..647aea543 100644 --- a/Zotlabs/Daemon/Cron.php +++ b/Zotlabs/Daemon/Cron.php @@ -3,6 +3,7 @@ namespace Zotlabs\Daemon; use Zotlabs\Lib\Config; +use Zotlabs\Lib\ObjCache; use Zotlabs\Lib\Libsync; use Zotlabs\Lib\Libzotdir; @@ -236,6 +237,73 @@ class Cron { if (!$restart) Master::Summon(array('Cronhooks')); + + // move as obj cache to fs + if (!Config::Get('system', 'as_objects_moved')) { + $results = dbq("select iconfig.*, item.mid from iconfig left join item on iid = item.id where cat = 'activitypub' and k = 'rawmsg' limit 300"); + if ($results) { + foreach ($results as $result) { + if (is_string($result['v'])) { + if (str_starts_with($result['v'], '{')) { + $result['v'] = json_decode($result['v'], true); + } + + elseif (str_starts_with($result['v'], 'json:')) { + $result['v'] = unserialise($result['v']); + } + + elseif (preg_match('|^a:[0-9]+:{.*}$|s', $result['v'])) { + $result['v'] = unserialize($result['v'], ['allowed_classes' => false]); + } + } + + if (is_array($result['v'])) { + ObjCache::Set($result['mid'], $result['v']); + } + + q("delete from iconfig where id = %d", + intval($result['id']) + ); + } + } + else { + Config::Set('system', 'as_objects_moved', 1); + } + } + + // move diaspora obj cache to fs + if (!Config::Get('system', 'diaspora_objects_moved')) { + $results = dbq("select iconfig.*, item.mid from iconfig left join item on iid = item.id where cat = 'diaspora' and k = 'fields' limit 300"); + if ($results) { + foreach ($results as $result) { + if (is_string($result['v'])) { + if (str_starts_with($result['v'], '{')) { + $result['v'] = json_decode($result['v'], true); + } + + elseif (str_starts_with($result['v'], 'json:')) { + $result['v'] = unserialise($result['v']); + } + + elseif (preg_match('|^a:[0-9]+:{.*}$|s', $result['v'])) { + $result['v'] = unserialize($result['v'], ['allowed_classes' => false]); + } + } + + if (is_array($result['v'])) { + ObjCache::Set($result['mid'], $result['v'], 'diaspora'); + } + + q("delete from iconfig where id = %d", + intval($result['id']) + ); + } + } + else { + Config::Set('system', 'diaspora_objects_moved', 1); + } + } + Config::Set('system', 'lastcron', datetime_convert()); //All done - clear the lockfile diff --git a/Zotlabs/Daemon/Notifier.php b/Zotlabs/Daemon/Notifier.php index 9fdb1defb..81a17d8fe 100644 --- a/Zotlabs/Daemon/Notifier.php +++ b/Zotlabs/Daemon/Notifier.php @@ -4,6 +4,8 @@ namespace Zotlabs\Daemon; use Zotlabs\Lib\Activity; use Zotlabs\Lib\Config; +use Zotlabs\Lib\IConfig; +use Zotlabs\Lib\ObjCache; use Zotlabs\Lib\Libzot; use Zotlabs\Lib\Queue; @@ -317,10 +319,15 @@ class Notifier { return; } - $m = get_iconfig($target_item, 'activitypub', 'signed_data'); + $m = ObjCache::Get($target_item['mid']); + + if (!$m) { + $m = IConfig::Get($target_item, 'activitypub', 'rawmsg'); + } + // Re-use existing signature unless the activity type changed to a Tombstone, which won't verify. if ($m && (!intval($target_item['item_deleted']))) { - self::$encoded_item = json_decode($m, true); + self::$encoded_item = $m; } else { $activity = Activity::encode_activity($target_item); diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php index 22dbaad84..9fca8e4a7 100644 --- a/Zotlabs/Lib/Activity.php +++ b/Zotlabs/Lib/Activity.php @@ -382,7 +382,12 @@ class Activity { if ($items) { $x = []; foreach ($items as $i) { - $m = IConfig::Get($i['id'], 'activitypub', 'rawmsg'); + $m = ObjCache::Get($i['mid']); + + if (!$m) { + $m = IConfig::Get($i['id'], 'activitypub', 'rawmsg'); + } + if ($m) { if (is_string($m)) $t = json_decode($m, true); @@ -501,8 +506,8 @@ class Activity { } } - $ret['id'] = ((strpos($i['mid'], 'http') === 0) ? $i['mid'] : z_root() . '/item/' . urlencode($i['mid'])); - $ret['diaspora:guid'] = $i['uuid']; + $ret['id'] = ((strpos($i['mid'], 'http') === 0) ? $i['mid'] : z_root() . '/item/' . urlencode($i['mid'])); + $ret['uuid'] = $i['uuid']; $images = []; $audios = []; @@ -829,8 +834,7 @@ class Activity { if ($iconfig && array_key_exists('iconfig', $item) && is_array($item['iconfig'])) { foreach ($item['iconfig'] as $att) { if ($att['sharing']) { - $value = ((is_string($att['v']) && preg_match('|^a:[0-9]+:{.*}$|s', $att['v'])) ? unserialize($att['v']) : $att['v']); - $ret[] = ['type' => 'PropertyValue', 'name' => 'zot.' . $att['cat'] . '.' . $att['k'], 'value' => $value]; + $ret[] = ['type' => 'PropertyValue', 'name' => 'zot.' . $att['cat'] . '.' . $att['k'], 'value' => unserialise($att['v'])]; } } } @@ -997,7 +1001,7 @@ class Activity { $ret['id'] = ((strpos($i['mid'], 'http') === 0) ? $i['mid'] : z_root() . '/activity/' . urlencode($i['mid'])); } - $ret['diaspora:guid'] = $i['uuid']; + $ret['uuid'] = $i['uuid']; if (!empty($i['title'])) $ret['name'] = html2plain(bbcode($i['title'])); @@ -2674,50 +2678,6 @@ class Activity { $s['item_private'] = 2; } - $ap_rawmsg = ''; - $diaspora_rawmsg = ''; - $raw_arr = []; - - $raw_arr = json_decode($act->raw, true); - - // This is a zot6 packet and the raw activitypub or diaspora message json - // is possibly available in the attachement. - if (array_key_exists('signed', $raw_arr) && isset($act->data['attachment']) && is_array($act->data['attachment'])) { - foreach($act->data['attachment'] as $a) { - if ( - isset($a['type']) && $a['type'] === 'PropertyValue' && - isset($a['name']) && $a['name'] === 'zot.activitypub.rawmsg' && - isset($a['value']) - ) { - $ap_rawmsg = $a['value']; - } - if ( - isset($a['type']) && $a['type'] === 'PropertyValue' && - isset($a['name']) && $a['name'] === 'zot.diaspora.fields' && - isset($a['value']) - ) { - $diaspora_rawmsg = $a['value']; - } - } - } - - - if (!$ap_rawmsg && array_key_exists('signed', $raw_arr)) { - // zap - $ap_rawmsg = json_encode($act->data, JSON_UNESCAPED_SLASHES); - } - - if ($ap_rawmsg) { - IConfig::Set($s, 'activitypub', 'rawmsg', $ap_rawmsg, 1); - } - elseif (!array_key_exists('signed', $raw_arr)) { - IConfig::Set($s, 'activitypub', 'rawmsg', $act->raw, 1); - } - - if ($diaspora_rawmsg) { - IConfig::Set($s, 'diaspora', 'fields', $diaspora_rawmsg, 1); - } - if ($act->raw_recips) { IConfig::Set($s, 'activitypub', 'recips', $act->raw_recips); } @@ -2726,6 +2686,8 @@ class Activity { IConfig::Set($s, 'event', 'timezone', $act->objprop('timezone'), true); } + ObjCache::Set($s['mid'], $act->data); + $hookinfo = [ 'act' => $act, 's' => $s @@ -2889,7 +2851,7 @@ class Activity { if (tgroup_check($channel['channel_id'], $item) && (!$is_child_node)) { // for forum deliveries, make sure we keep a copy of the signed original - IConfig::Set($item, 'activitypub', 'rawmsg', $act->raw, 1); + ObjCache::Set($item['mid'], $act->data); $allowed = true; } @@ -3746,8 +3708,6 @@ class Activity { 'conversation' => 'ostatus:conversation', - 'guid' => 'diaspora:guid', - 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers', 'Hashtag' => 'as:Hashtag', diff --git a/Zotlabs/Lib/Config.php b/Zotlabs/Lib/Config.php index cd8b08991..139affa09 100644 --- a/Zotlabs/Lib/Config.php +++ b/Zotlabs/Lib/Config.php @@ -132,8 +132,8 @@ class Config { $value = App::$config[$family][$key]; if (! is_array($value)) { - if (substr($value, 0, 5) == 'json:') { - return json_decode(substr($value, 5), true); + if (str_starts_with($value, 'json:')) { + return unserialise($value); } else if (preg_match('|^a:[0-9]+:{.*}$|s', $value)) { // Unserialize in inherently unsafe. Try to mitigate by not // allowing unserializing objects. Only kept for backwards diff --git a/Zotlabs/Lib/IConfig.php b/Zotlabs/Lib/IConfig.php index 3540c2b24..3e3f783f5 100644 --- a/Zotlabs/Lib/IConfig.php +++ b/Zotlabs/Lib/IConfig.php @@ -34,8 +34,20 @@ class IConfig { if(is_array($item) && array_key_exists('iconfig',$item) && is_array($item['iconfig'])) { foreach($item['iconfig'] as $c) { - if (isset($c['iid']) && $c['iid'] == $iid && isset($c['cat']) && $c['cat'] == $family && isset($c['k']) && $c['k'] == $key) + if (isset($c['iid']) && $c['iid'] == $iid && isset($c['cat']) && $c['cat'] == $family && isset($c['k']) && $c['k'] == $key) { + if (is_string($c['v'])) { + if (str_starts_with($c['v'], 'json:')) { + $c['v'] = unserialise($c['v']); + } else if (preg_match('|^a:[0-9]+:{.*}$|s', $c['v'])) { + // Unserialize in inherently unsafe. Try to mitigate by not + // allowing unserializing objects. Only kept for backwards + // compatibility. JSON serialization should be prefered. + $c['v'] = unserialize($c['v'], ['allowed_classes' => false]); + } + } + return $c['v']; + } } } @@ -44,12 +56,24 @@ class IConfig { dbesc($family), dbesc($key) ); + if($r) { - $r[0]['v'] = ((preg_match('|^a:[0-9]+:{.*}$|s',$r[0]['v'])) ? unserialize($r[0]['v']) : $r[0]['v']); - if($is_item) + if (str_starts_with($r[0]['v'], 'json:')) { + $r[0]['v'] = unserialise($r[0]['v']); + } else if (preg_match('|^a:[0-9]+:{.*}$|s', $r[0]['v'])) { + // Unserialize in inherently unsafe. Try to mitigate by not + // allowing unserializing objects. Only kept for backwards + // compatibility. JSON serialization should be prefered. + $r[0]['v'] = unserialize($r[0]['v'], ['allowed_classes' => false]); + } + + if ($is_item) { $item['iconfig'][] = $r[0]; + } + return $r[0]['v']; } + return $default; } @@ -73,7 +97,7 @@ class IConfig { static public function Set(&$item, $family, $key, $value, $sharing = false) { - $dbvalue = ((is_array($value)) ? serialize($value) : $value); + $dbvalue = ((is_array($value)) ? serialise($value) : $value); $dbvalue = ((is_bool($dbvalue)) ? intval($dbvalue) : $dbvalue); $is_item = false; diff --git a/Zotlabs/Lib/Libzot.php b/Zotlabs/Lib/Libzot.php index 08a3f27c9..e3fa6e9d3 100644 --- a/Zotlabs/Lib/Libzot.php +++ b/Zotlabs/Lib/Libzot.php @@ -6,7 +6,6 @@ use App; use Zotlabs\Access\PermissionLimits; use Zotlabs\Access\Permissions; use Zotlabs\Daemon\Master; -use Zotlabs\Lib\Config; use Zotlabs\Web\HTTPSig; require_once('include/crypto.php'); @@ -1298,10 +1297,6 @@ class Libzot { $item['comment_policy'] = 'authenticated'; } - if (isset($AS->meta['signed_data']) && $AS->meta['signed_data']) { - IConfig::Set($item, 'activitypub', 'signed_data', $AS->meta['signed_data'], false); - } - logger('Activity received: ' . print_r($item, true), LOGGER_DATA, LOG_DEBUG); logger('Activity recipients: ' . print_r($deliveries, true), LOGGER_DATA, LOG_DEBUG); @@ -2144,10 +2139,9 @@ class Libzot { } if (isset($AS->meta['signed_data'])) { - IConfig::Set($arr, 'activitypub', 'signed_data', $AS->meta['signed_data'], false); $j = json_decode($AS->meta['signed_data'], true); if ($j) { - IConfig::Set($arr, 'activitypub', 'rawmsg', json_encode(JSalmon::unpack($j['data'])), true); + ObjCache::Set($arr['mid'], json_encode(JSalmon::unpack($j['data']))); } } diff --git a/Zotlabs/Lib/ObjCache.php b/Zotlabs/Lib/ObjCache.php new file mode 100644 index 000000000..618522cf2 --- /dev/null +++ b/Zotlabs/Lib/ObjCache.php @@ -0,0 +1,40 @@ +<?php + +namespace Zotlabs\Lib; + +class ObjCache +{ + public static function Get($path, $type = 'as') + { + if (!$path) { + return []; + } + + $localpath = Hashpath::path($path, 'store/[data]/[obj]/' . $type, 2, alg: 'sha256'); + if (file_exists($localpath)) { + return unserialise(file_get_contents($localpath)); + } + + return []; + } + + public static function Set($path, $content, $type = 'as') { + if (!$path) { + return; + } + + $localpath = Hashpath::path($path, 'store/[data]/[obj]/' . $type, 2, alg: 'sha256'); + file_put_contents($localpath, serialise($content)); + } + + public static function Delete($path, $type = 'as') { + if (!$path) { + return; + } + + $localpath = Hashpath::path($path, 'store/[data]/[obj]/' . $type, 2, alg: 'sha256'); + if (file_exists($localpath)) { + unlink($localpath); + } + } +} diff --git a/Zotlabs/Module/Viewsrc.php b/Zotlabs/Module/Viewsrc.php index cfc184a9d..c3d8d96de 100644 --- a/Zotlabs/Module/Viewsrc.php +++ b/Zotlabs/Module/Viewsrc.php @@ -1,7 +1,9 @@ <?php namespace Zotlabs\Module; - +use Zotlabs\Lib\Activity; +use Zotlabs\Lib\IConfig; +use Zotlabs\Lib\ObjCache; class Viewsrc extends \Zotlabs\Web\Controller { @@ -28,13 +30,16 @@ class Viewsrc extends \Zotlabs\Web\Controller { $item_normal = item_normal_search(); if(local_channel() && $item_id) { - $r = q("select id, mid, uuid, item_flags, mimetype, item_obscured, body, llink, plink from item where uid in (%d , %d) and id = %d $item_normal limit 1", + $r = q("select * from item where uid in (%d , %d) and id = %d $item_normal limit 1", intval(local_channel()), intval($sys['channel_id']), intval($item_id) ); if($r) { + xchan_query($r, true); + $r = fetch_post_tags($r); + if(intval($r[0]['item_obscured'])) $dload = true; @@ -45,9 +50,35 @@ class Viewsrc extends \Zotlabs\Web\Controller { killme(); } + $cached = true; + + $obj = ObjCache::Get($r[0]['mid']); + + if (!$obj) { + $obj = IConfig::Get($r[0], 'activitypub', 'rawmsg'); + } + + if (in_array($r[0]['owner']['xchan_network'], ['diaspora'])) { + $obj = ObjCache::Get($r[0]['mid'], 'diaspora'); + + if (!$obj) { + $obj = IConfig::Get($r[0], 'diaspora', 'fields'); + } + } + + if (!$obj) { + $cached = false; + $obj = Activity::encode_activity($r[0]); + } + + if ($obj) { + $content = (($cached) ? 'Cached: ' : '') . '<pre>' . escape_tags(json_encode($obj, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)) . '</pre>'; + } + else { + $content = escape_tags($r[0]['body']); + } - $content = escape_tags($r[0]['body']); - $o = (($json) ? json_encode($content) : $content); + $o = (($json) ? json_encode($content) : str_replace("\n", '<br>', $content)); } } diff --git a/include/items.php b/include/items.php index 993fc86b5..b5c9bd5f7 100644 --- a/include/items.php +++ b/include/items.php @@ -10,6 +10,7 @@ use Zotlabs\Lib\Enotify; use Zotlabs\Lib\MarkdownSoap; use Zotlabs\Lib\MessageFilter; use Zotlabs\Lib\ThreadListener; +use Zotlabs\Lib\ObjCache; use Zotlabs\Lib\IConfig; use Zotlabs\Lib\PConfig; use Zotlabs\Lib\Activity; @@ -4179,6 +4180,15 @@ function delete_item_lowlevel($item, $stage = DROPITEM_NORMAL) { intval($item['id']) ); + $n = q("SELECT count(id) AS total FROM item WHERE mid = '%s'", + dbesc($item['mid']) + ); + + if (!$n[0]['total']) { + ObjCache::Delete($item['mid']); + ObjCache::Delete($item['mid'], 'diaspora'); + } + q("delete from term where oid = %d and otype = %d", intval($item['id']), intval(TERM_OBJ_POST) diff --git a/include/text.php b/include/text.php index b66c2abcd..05700072a 100644 --- a/include/text.php +++ b/include/text.php @@ -4098,7 +4098,7 @@ function unserialise($x) { if (is_array($x)) { return $x; } - $y = ((substr($x,0,5) === 'json:') ? json_decode(substr($x,5),true) : ''); + $y = ((str_starts_with($x, 'json:')) ? json_decode(substr($x, 5), true) : ''); return ((is_array($y)) ? $y : $x); } |