diff options
| -rw-r--r-- | Zotlabs/Daemon/Fetchparents.php | 9 | ||||
| -rw-r--r-- | Zotlabs/Lib/ASCache.php | 33 | ||||
| -rw-r--r-- | Zotlabs/Lib/Activity.php | 335 | ||||
| -rw-r--r-- | Zotlabs/Lib/ActivityStreams.php | 21 | ||||
| -rw-r--r-- | Zotlabs/Lib/Cache.php | 16 | ||||
| -rw-r--r-- | boot.php | 1 |
6 files changed, 125 insertions, 290 deletions
diff --git a/Zotlabs/Daemon/Fetchparents.php b/Zotlabs/Daemon/Fetchparents.php index 4daf8c019..b00acdfbf 100644 --- a/Zotlabs/Daemon/Fetchparents.php +++ b/Zotlabs/Daemon/Fetchparents.php @@ -14,8 +14,8 @@ class Fetchparents { return; } - $channel = channelx_by_n(intval($argv[1])); - if (!$channel) { + $channels = explode(',', $argv[1]); + if (!$channels) { return; } @@ -31,7 +31,10 @@ class Fetchparents { $force = $argv[4] ?? false; - Activity::fetch_and_store_parents($channel, $observer_hash, $mid, null, $force); + foreach ($channels as $channel_id) { + $channel = channelx_by_n($channel_id); + Activity::fetch_and_store_parents($channel, $observer_hash, $mid, null, $force); + } return; diff --git a/Zotlabs/Lib/ASCache.php b/Zotlabs/Lib/ASCache.php new file mode 100644 index 000000000..63bd73ea7 --- /dev/null +++ b/Zotlabs/Lib/ASCache.php @@ -0,0 +1,33 @@ +<?php /** @file */ + +namespace Zotlabs\Lib; + + /** + * A wrapper for the cache api + */ + +class ASCache { + public static function isEnabled() { + return Config::Get('system', 'as_object_cache_enabled', false); + } + + public static function getAge() { + return Config::Get('system', 'as_object_cache_time', '10 MINUTE'); + } + + public static function Get($key) { + if (!self::isEnabled()) { + return; + } + + return Cache::get($key, self::getAge()); + } + + public static function Set($key, $value) { + if (!self::isEnabled()) { + return; + } + + Cache::set($key, $value); + } +} diff --git a/Zotlabs/Lib/Activity.php b/Zotlabs/Lib/Activity.php index c2049bb7b..a225e551b 100644 --- a/Zotlabs/Lib/Activity.php +++ b/Zotlabs/Lib/Activity.php @@ -153,6 +153,7 @@ class Activity { } else { logger('logger_stats_data cmd:Activity_fetch' . ' start:' . $start_timestamp . ' ' . 'end:' . microtime(true) . ' meta:' . $url . '#' . random_string(16)); + btlogger('activity fetch'); } return json_decode($x['body'], true); @@ -363,7 +364,7 @@ class Activity { if ($items) { $x = []; foreach ($items as $i) { - $m = get_iconfig($i['id'], 'activitypub', 'rawmsg'); + $m = IConfig::Get($i['id'], 'activitypub', 'rawmsg'); if ($m) { if (is_string($m)) $t = json_decode($m, true); @@ -498,7 +499,7 @@ class Activity { // set this for descendants even if the current item is not private // because it may have been relayed from a private item. - $token = get_iconfig($i, 'ocap', 'relay'); + $token = IConfig::Get($i, 'ocap', 'relay'); if ($token && $has_images) { $matches_processed = []; for ($n = 0; $n < count($images); $n++) { @@ -1083,11 +1084,11 @@ class Activity { $ret['type'] = 'Person'; if ($c) { - if (get_pconfig($c['channel_id'], 'system', 'group_actor')) { + if (PConfig::Get($c['channel_id'], 'system', 'group_actor')) { $ret['type'] = 'Group'; } - $ret['manuallyApprovesFollowers'] = ((get_pconfig($c['channel_id'], 'system', 'autoperms')) ? false : true); + $ret['manuallyApprovesFollowers'] = ((PConfig::Get($c['channel_id'], 'system', 'autoperms')) ? false : true); } $ret['id'] = $id; @@ -1396,7 +1397,7 @@ class Activity { } } - $role = get_pconfig($channel['channel_id'], 'system', 'permissions_role', 'personal'); + $role = PConfig::Get($channel['channel_id'], 'system', 'permissions_role', 'personal'); $x = PermissionRoles::role_perms($role); $their_perms = Permissions::FilledPerms($x['perms_connect']); @@ -1417,7 +1418,7 @@ class Activity { // We've already approved them or followed them first // Send an Accept back to them - set_abconfig($channel['channel_id'], $person_obj['id'], 'pubcrawl', 'their_follow_id', $their_follow_id); + AbConfig::Set($channel['channel_id'], $person_obj['id'], 'pubcrawl', 'their_follow_id', $their_follow_id); Master::Summon(['Notifier', 'permission_accept', $contact['abook_id']]); return; @@ -1429,7 +1430,7 @@ class Activity { if(in_array($k, ['send_stream', 'post_wall'])) { continue; // Those will be set once we accept their follow request } - set_abconfig($channel['channel_id'], $contact['abook_xchan'], 'their_perms', $k, $v); + AbConfig::Set($channel['channel_id'], $contact['abook_xchan'], 'their_perms', $k, $v); } $abook_instance = $contact['abook_instance']; @@ -1463,7 +1464,7 @@ class Activity { // From here on out we assume a Follow activity to somebody we have no existing relationship with - set_abconfig($channel['channel_id'], $person_obj['id'], 'pubcrawl', 'their_follow_id', $their_follow_id); + AbConfig::Set($channel['channel_id'], $person_obj['id'], 'pubcrawl', 'their_follow_id', $their_follow_id); // The xchan should have been created by actor_store() above @@ -1481,7 +1482,7 @@ class Activity { $my_perms = $p['perms']; $automatic = $p['automatic']; - $closeness = get_pconfig($channel['channel_id'], 'system', 'new_abook_closeness', 80); + $closeness = PConfig::Get($channel['channel_id'], 'system', 'new_abook_closeness', 80); $r = abook_store_lowlevel( [ @@ -1500,11 +1501,11 @@ class Activity { if ($my_perms) foreach ($my_perms as $k => $v) - set_abconfig($channel['channel_id'], $ret['xchan_hash'], 'my_perms', $k, $v); + AbConfig::Set($channel['channel_id'], $ret['xchan_hash'], 'my_perms', $k, $v); if ($their_perms) foreach ($their_perms as $k => $v) - set_abconfig($channel['channel_id'], $ret['xchan_hash'], 'their_perms', $k, $v); + AbConfig::Set($channel['channel_id'], $ret['xchan_hash'], 'their_perms', $k, $v); if ($r) { logger("New ActivityPub follower for {$channel['channel_name']}"); @@ -1540,7 +1541,7 @@ class Activity { unset($clone['abook_account']); unset($clone['abook_channel']); - $abconfig = load_abconfig($channel['channel_id'], $clone['abook_xchan']); + $abconfig = AbConfig::Load($channel['channel_id'], $clone['abook_xchan']); if ($abconfig) $clone['abconfig'] = $abconfig; @@ -1581,7 +1582,7 @@ class Activity { ); if ($r) { // remove all permissions they provided - del_abconfig($channel['channel_id'], $r[0]['xchan_hash'], 'system', 'their_perms'); + AbConfig::Delete($channel['channel_id'], $r[0]['xchan_hash'], 'system', 'their_perms'); } } @@ -1881,251 +1882,6 @@ class Activity { return (($a_width > $b_width) ? -1 : 1); } - static function create_note($channel, $observer_hash, $act) { - - $s = []; - $is_sys_channel = is_sys_channel($channel['channel_id']); - $parent = ((array_key_exists('inReplyTo', $act->obj)) ? urldecode($act->obj['inReplyTo']) : ''); - - if ($parent) { - - $r = q("select * from item where uid = %d and ( mid = '%s' or mid = '%s' ) limit 1", - intval($channel['channel_id']), - dbesc($parent), - dbesc(basename($parent)) - ); - - if (!$r) { - logger('parent not found.'); - return; - } - - if ($r[0]['owner_xchan'] === $channel['channel_hash']) { - if (!perm_is_allowed($channel['channel_id'], $observer_hash, 'send_stream') && !$is_sys_channel) { - logger('no comment permission.'); - return; - } - } - - $s['parent_mid'] = $r[0]['mid']; - $s['owner_xchan'] = $r[0]['owner_xchan']; - $s['author_xchan'] = $observer_hash; - - } - else { - if (!perm_is_allowed($channel['channel_id'], $observer_hash, 'send_stream') && !$is_sys_channel) { - logger('no send_stream permission'); - return; - } - $s['owner_xchan'] = $s['author_xchan'] = $observer_hash; - } - - if ($act->recips && (!in_array(ACTIVITY_PUBLIC_INBOX, $act->recips))) - $s['item_private'] = 1; - - - if (array_key_exists('directMessage', $act->obj) && intval($act->obj['directMessage'])) { - $s['item_private'] = 2; - } - - if (intval($s['item_private']) === 2) { - if (!perm_is_allowed($channel['channel_id'], $observer_hash, 'post_mail')) { - logger('no post_mail permission'); - return; - } - } - - $content = self::get_content($act->obj); - - if (!$content) { - logger('no content'); - return; - } - - $s['aid'] = $channel['channel_account_id']; - $s['uid'] = $channel['channel_id']; - - // Make sure we use the zot6 identity where applicable - - $s['author_xchan'] = self::find_best_identity($s['author_xchan']); - $s['owner_xchan'] = self::find_best_identity($s['owner_xchan']); - - if (!$s['author_xchan']) { - logger('No author: ' . print_r($act, true)); - } - - if (!$s['owner_xchan']) { - logger('No owner: ' . print_r($act, true)); - } - - if (!$s['author_xchan'] || !$s['owner_xchan']) - return; - - $s['mid'] = urldecode($act->obj['id']); - $s['uuid'] = $act->obj['diaspora:guid']; - $s['plink'] = urldecode($act->obj['id']); - - - if ($act->data['published']) { - $s['created'] = datetime_convert('UTC', 'UTC', $act->data['published']); - } - elseif ($act->obj['published']) { - $s['created'] = datetime_convert('UTC', 'UTC', $act->obj['published']); - } - if ($act->data['updated']) { - $s['edited'] = datetime_convert('UTC', 'UTC', $act->data['updated']); - } - elseif ($act->obj['updated']) { - $s['edited'] = datetime_convert('UTC', 'UTC', $act->obj['updated']); - } - if ($act->data['expires']) { - $s['expires'] = datetime_convert('UTC', 'UTC', $act->data['expires']); - } - elseif ($act->obj['expires']) { - $s['expires'] = datetime_convert('UTC', 'UTC', $act->obj['expires']); - } - - if (!$s['created']) - $s['created'] = datetime_convert(); - - if (!$s['edited']) - $s['edited'] = $s['created']; - - - if (!$s['parent_mid']) - $s['parent_mid'] = $s['mid']; - - - $s['title'] = self::bb_content($content, 'name'); - $s['summary'] = self::bb_content($content, 'summary'); - $s['body'] = self::bb_content($content, 'content'); - $s['verb'] = ACTIVITY_POST; - $s['obj_type'] = ACTIVITY_OBJ_NOTE; - - $generator = $act->get_property_obj('generator'); - if (!$generator) - $generator = $act->get_property_obj('generator', $act->obj); - - if ($generator && array_key_exists('type', $generator) - && in_array($generator['type'], ['Application', 'Service']) && array_key_exists('name', $generator)) { - $s['app'] = escape_tags($generator['name']); - } - - if ($channel['channel_system']) { - $incl = get_config('system','pubstream_incl'); - $excl = get_config('system','pubstream_excl'); - - if(($incl || $excl) && !MessageFilter::evaluate($s, $incl, $excl)) { - logger('post is filtered'); - return; - } - } - - $abook = q("select * from abook where (abook_xchan = '%s' OR abook_xchan = '%s') and abook_channel = %d ", - dbesc($s['author_xchan']), - dbesc($s['owner_xchan']), - intval($channel['channel_id']) - ); - - if ($abook) { - if (!post_is_importable($channel['channel_id'], $s, $abook)) { - logger('post is filtered'); - return; - } - } - - if ($act->obj['conversation']) { - set_iconfig($s, 'ostatus', 'conversation', $act->obj['conversation'], 1); - } - - $a = self::decode_taxonomy($act->obj); - if ($a) { - $s['term'] = $a; - } - - $a = self::decode_attachment($act->obj); - if ($a) { - $s['attach'] = $a; - } - - if ($act->obj['type'] === 'Note' && $s['attach']) { - $s['body'] .= self::bb_attach($s['attach'], $s['body']); - } - - // we will need a hook here to extract magnet links e.g. peertube - // right now just link to the largest mp4 we find that will fit in our - // standard content region - - if ($act->obj['type'] === 'Video') { - - $vtypes = [ - 'video/mp4', - 'video/ogg', - 'video/webm' - ]; - - $mps = []; - if (array_key_exists('url', $act->obj) && is_array($act->obj['url'])) { - foreach ($act->obj['url'] as $vurl) { - if (in_array($vurl['mimeType'], $vtypes)) { - if (!array_key_exists('width', $vurl)) { - $vurl['width'] = 0; - } - $mps[] = $vurl; - } - } - } - if ($mps) { - usort($mps, [__CLASS__, 'vid_sort']); - foreach ($mps as $m) { - if (intval($m['width']) < 500) { - $s['body'] .= "\n\n" . '[video]' . $m['href'] . '[/video]'; - break; - } - } - } - } - - set_iconfig($s, 'activitypub', 'recips', $act->raw_recips); - if ($parent) { - set_iconfig($s, 'activitypub', 'rawmsg', $act->raw, 1); - } - - $x = null; - - $r = q("select created, edited from item where mid = '%s' and uid = %d limit 1", - dbesc($s['mid']), - intval($s['uid']) - ); - if ($r) { - if ($s['edited'] > $r[0]['edited']) { - $x = item_store_update($s); - } - else { - return; - } - } - else { - $x = item_store($s); - } - - if (is_array($x) && $x['item_id']) { - if ($parent) { - if ($s['owner_xchan'] === $channel['channel_hash']) { - // We are the owner of this conversation, so send all received comments back downstream - Master::Summon(['Notifier', 'comment-import', $x['item_id']]); - } - $r = q("select * from item where id = %d limit 1", - intval($x['item_id']) - ); - if ($r) { - send_status_notifications($x['item_id'], $r[0]); - } - } - sync_an_item($channel['channel_id'], $x['item_id']); - } - - } static function get_actor_bbmention($id) { @@ -2305,7 +2061,16 @@ class Activity { $content = self::get_content($act->obj); } - $s['mid'] = $act->objprop('id') ?: $act->obj; + $s['mid'] = $act->objprop('id'); + + if (!$s['mid'] && is_string($act->obj)) { + $s['mid'] = $act->obj; + } + + // pleroma fetched activities + if (!$s['mid'] && isset($act->obj['data']['id'])) { + $s['mid'] = $act->obj['data']['id']; + } if (!$s['mid']) { return false; @@ -2883,22 +2648,22 @@ class Activity { } if ($ap_rawmsg) { - set_iconfig($s, 'activitypub', 'rawmsg', $ap_rawmsg, 1); + IConfig::Set($s, 'activitypub', 'rawmsg', $ap_rawmsg, 1); } elseif (!array_key_exists('signed', $raw_arr)) { - set_iconfig($s, 'activitypub', 'rawmsg', $act->raw, 1); + IConfig::Set($s, 'activitypub', 'rawmsg', $act->raw, 1); } if ($diaspora_rawmsg) { - set_iconfig($s, 'diaspora', 'fields', $diaspora_rawmsg, 1); + IConfig::Set($s, 'diaspora', 'fields', $diaspora_rawmsg, 1); } if ($act->raw_recips) { - set_iconfig($s, 'activitypub', 'recips', $act->raw_recips); + IConfig::Set($s, 'activitypub', 'recips', $act->raw_recips); } if ($act->objprop('type') === 'Event' && $act->objprop('timezone')) { - set_iconfig($s, 'event', 'timezone', $act->objprop('timezone'), true); + IConfig::Set($s, 'event', 'timezone', $act->objprop('timezone'), true); } $hookinfo = [ @@ -2947,14 +2712,18 @@ class Activity { } if ($fetch_parents) { - Master::Summon(['Fetchparents', $channel['channel_id'], $observer_hash, $item['mid'], $force]); + // Cache::set($item['mid'], 'json:' . $act->raw); + App::$cache['fetch_objects'][$item['mid']]['channels'][] = $channel['channel_id']; + App::$cache['fetch_objects'][$item['mid']]['force'] = intval($force); + + //Master::Summon(['Fetchparents', $channel['channel_id'], $observer_hash, $item['mid'], $force]); //self::fetch_and_store_parents($channel, $observer_hash, $item, $act, $force); return; } - - logger('no parent'); - return; } + + logger('no parent'); + return; } @@ -2998,7 +2767,7 @@ class Activity { }*/ if (!$allowed) { - if (get_pconfig($channel['channel_id'], 'system', 'moderate_unsolicited_comments') && $item['obj_type'] !== 'Answer') { + if (PConfig::Get($channel['channel_id'], 'system', 'moderate_unsolicited_comments') && $item['obj_type'] !== 'Answer') { $item['item_blocked'] = ITEM_MODERATED; $allowed = true; } @@ -3043,7 +2812,7 @@ class Activity { if (tgroup_check($channel['channel_id'], $item) && (!$is_child_node)) { // for forum deliveries, make sure we keep a copy of the signed original - set_iconfig($item, 'activitypub', 'rawmsg', $act->raw, 1); + IConfig::Set($item, 'activitypub', 'rawmsg', $act->raw, 1); $allowed = true; } @@ -3103,6 +2872,7 @@ class Activity { $item['author_xchan'] = self::find_best_identity($item['author_xchan']); $item['owner_xchan'] = self::find_best_identity($item['owner_xchan']); + $item['source_xchan'] = ((!empty($item['source_xchan'])) ? self::find_best_identity($item['source_xchan']) : ''); if (!$item['author_xchan']) { logger('No author: ' . print_r($act, true)); @@ -3116,8 +2886,8 @@ class Activity { return; if ($channel['channel_system']) { - $incl = get_config('system','pubstream_incl'); - $excl = get_config('system','pubstream_excl'); + $incl = Config::Get('system','pubstream_incl'); + $excl = Config::Get('system','pubstream_excl'); if(($incl || $excl) && !MessageFilter::evaluate($item, $incl, $excl)) { logger('post is filtered'); @@ -3125,9 +2895,10 @@ class Activity { } } - $abook = q("select * from abook where ( abook_xchan = '%s' OR abook_xchan = '%s') and abook_channel = %d ", + $abook = q("select * from abook where ( abook_xchan = '%s' OR abook_xchan = '%s' OR abook_xchan = '%s') and abook_channel = %d ", dbesc($item['author_xchan']), dbesc($item['owner_xchan']), + dbesc($item['source_xchan']), intval($channel['channel_id']) ); @@ -3139,13 +2910,13 @@ class Activity { } if (array_key_exists('conversation', $act->obj)) { - set_iconfig($item, 'ostatus', 'conversation', $act->obj['conversation'], 1); + IConfig::Set($item, 'ostatus', 'conversation', $act->obj['conversation'], 1); } // This isn't perfect but the best we can do for now. $item['comment_policy'] = ((isset($act->data['commentPolicy'])) ? $act->data['commentPolicy'] : 'authenticated'); - set_iconfig($item, 'activitypub', 'recips', $act->raw_recips); + IConfig::Set($item, 'activitypub', 'recips', $act->raw_recips); if (intval($act->sigok)) { $item['item_verified'] = 1; @@ -3311,10 +3082,16 @@ class Activity { $i = 0; while ($current_item['parent_mid'] !== $current_item['mid']) { - $n = self::fetch($current_item['parent_mid'], $channel); - - if (!$n) { - break; + $cached = ASCache::Get($current_item['parent_mid']); + if ($cached) { + $n = unserialise($cached); + } + else { + $n = self::fetch($current_item['parent_mid'], $channel); + if (!$n) { + break; + } + ASCache::Set($current_item['parent_mid'], serialise($n)); } $a = new ActivityStreams($n); diff --git a/Zotlabs/Lib/ActivityStreams.php b/Zotlabs/Lib/ActivityStreams.php index 0770f2040..3749126d3 100644 --- a/Zotlabs/Lib/ActivityStreams.php +++ b/Zotlabs/Lib/ActivityStreams.php @@ -90,6 +90,15 @@ class ActivityStreams { // Attempt to assemble an Activity from what we were given. if ($this->is_valid()) { $this->id = $this->get_property_obj('id'); + + if (!$this->id) { + logger('Data with mmissing id: ' . print_r($this->data, true)); + return; + } + + // cache for future use + ASCache::Set($this->id, 'json:' . $this->raw); + $this->type = $this->get_primary_type(); $this->actor = $this->get_actor('actor', '', ''); $this->obj = $this->get_compound_property('object'); @@ -394,12 +403,22 @@ class ActivityStreams { $x = $this->get_property_obj($property, $base, $namespace); if ($this->is_url($x)) { - $y = $this->fetch_property($x); + $cached = ASCache::Get($x); + if ($cached) { + $y = unserialise($cached); + } + else { + $y = $this->fetch_property($x); + if ($y) { + ASCache::Set($x, serialise($y)); + } + } if (is_array($y)) { $x = $y; } } + // verify and unpack JSalmon signature if present if (is_array($x) && array_key_exists('signed', $x)) { diff --git a/Zotlabs/Lib/Cache.php b/Zotlabs/Lib/Cache.php index 60bf64611..f3f520496 100644 --- a/Zotlabs/Lib/Cache.php +++ b/Zotlabs/Lib/Cache.php @@ -17,8 +17,8 @@ class Cache { */ public static function get($key, $age = '') { - - $hash = hash('whirlpool',$key); +// $hash = hash('whirlpool',$key); + $hash = uuid_from_url($key); $r = q("SELECT v FROM cache WHERE k = '%s' AND updated > %s - INTERVAL %s LIMIT 1", dbesc($hash), @@ -32,23 +32,25 @@ class Cache { } public static function set($key,$value) { +// $hash = hash('whirlpool',$key); + $hash = uuid_from_url($key); - $hash = hash('whirlpool',$key); - - $r = q("SELECT * FROM cache WHERE k = '%s' limit 1", + $r = q("SELECT * FROM cache WHERE k = '%s' LIMIT 1", dbesc($hash) ); if($r) { q("UPDATE cache SET v = '%s', updated = '%s' WHERE k = '%s'", dbesc($value), dbesc(datetime_convert()), - dbesc($hash)); + dbesc($hash) + ); } else { q("INSERT INTO cache (k, v, updated) VALUES ('%s', '%s', '%s')", dbesc($hash), dbesc($value), - dbesc(datetime_convert())); + dbesc(datetime_convert()) + ); } } } @@ -804,6 +804,7 @@ class App { public static $is_sys = false; public static $nav_sel; public static $comanche; + public static $cache = []; // general purpose cache public static $channel_links; |