From 0350b76d851977ccd6eed4278738346f0febbf96 Mon Sep 17 00:00:00 2001 From: friendica Date: Sat, 13 Sep 2014 16:00:09 -0700 Subject: some backend work for the remaining missing bits of mod_hubman - this is still a fair ways from being complete and is not ready for prime time. Basically we'll let a channel send out a public message saying "these are my currently approved locations" and anything that isn't in the list will be marked deleted. We'll send out this message when locations change somehow - either through direct personal involvement (hub revoke, change primary, channel import) or during a system rename or "find bad/obsolete hublocs" activity. This way we won't have clones sending back location info we just got rid of and re-importing the bad entries. --- include/hubloc.php | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index 566875ce9..d51e38eae 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -42,4 +42,58 @@ function prune_hub_reinstalls() { } } } -} \ No newline at end of file +} + +function remove_obsolete_hublocs() { + + // Get rid of any hublocs which are ours but aren't valid anymore - + // e.g. they point to a different and perhaps transient URL that we aren't using. + + // I need to stress that this shouldn't happen. fix_system_urls() fixes hublocs + // when it discovers the URL has changed. So it's unclear how we could end up + // with URLs pointing to the old site name. But it happens. This may be an artifact + // of an old bug or maybe a regression in some newer code. In any event, they + // mess up communications and we have to take action if we find any. + + // First make sure we have any hublocs (at all) with this URL and sitekey. + // We don't want to perform this operation while somebody is in the process + // of renaming their hub or installing certs. + + $r = q("select hubloc_id from hubloc where hubloc_url = '%s' and hubloc_sitekey = '%s'", + dbesc(z_root()), + dbesc(get_config('system','pubkey')) + ); + if((! $r) || (! count($r))) + return; + + // Good. We have at least one valid hubloc. + + // Do we have any invalid ones? + + $r = q("select hubloc_id from hubloc where hubloc_sitekey = '%s' and hubloc_url != '%s'", + dbesc(get_config('system','pubkey')), + dbesc(z_root()) + ); + if(! $r) + return; + + logger('remove_obsolete_hublocs: removing ' . count($r) . ' hublocs.'); + + // We've got invalid hublocs. Get rid of them. + + $r = q("delete from hubloc where hubloc_sitekey = '%s' and hubloc_url != '%s'", + dbesc(get_config('system','pubkey')), + dbesc(z_root()) + ); + + // We should probably tell everybody... But we don't have an easy way to do this + // for the entire site. We'd have to do a channel at a time. + // They will find out anyway - it just might take a little while. + + // FIXME we probably also need to check that the sys channel has a valid hubloc + // and re-create it if it doesn't. + +} + + + \ No newline at end of file -- cgit v1.2.3 From 0dc33900b6799681683fd02f597e95acbd58e960 Mon Sep 17 00:00:00 2001 From: friendica Date: Sun, 14 Sep 2014 22:19:19 -0700 Subject: provide a way to sync locations and get rid of bogus hublocs, now implemented --- include/hubloc.php | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index d51e38eae..cdc9de4af 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -46,6 +46,8 @@ function prune_hub_reinstalls() { function remove_obsolete_hublocs() { + logger('remove_obsolete_hublocs',LOGGER_DEBUG); + // Get rid of any hublocs which are ours but aren't valid anymore - // e.g. they point to a different and perhaps transient URL that we aren't using. @@ -66,7 +68,9 @@ function remove_obsolete_hublocs() { if((! $r) || (! count($r))) return; - // Good. We have at least one valid hubloc. + $channels = array(); + + // Good. We have at least one *valid* hubloc. // Do we have any invalid ones? @@ -74,25 +78,38 @@ function remove_obsolete_hublocs() { dbesc(get_config('system','pubkey')), dbesc(z_root()) ); + $p = q("select hubloc_id from hubloc where hubloc_sitekey != '%s' and hubloc_url = '%s'", + dbesc(get_config('system','pubkey')), + dbesc(z_root()) + ); + if(is_array($r) && is_array($p)) + $r = array_merge($r,$p); + if(! $r) return; - logger('remove_obsolete_hublocs: removing ' . count($r) . ' hublocs.'); - // We've got invalid hublocs. Get rid of them. - $r = q("delete from hubloc where hubloc_sitekey = '%s' and hubloc_url != '%s'", - dbesc(get_config('system','pubkey')), - dbesc(z_root()) - ); + logger('remove_obsolete_hublocs: removing ' . count($r) . ' hublocs.'); - // We should probably tell everybody... But we don't have an easy way to do this - // for the entire site. We'd have to do a channel at a time. - // They will find out anyway - it just might take a little while. + $interval = ((get_config('system','delivery_interval') !== false) + ? intval(get_config('system','delivery_interval')) : 2 ); - // FIXME we probably also need to check that the sys channel has a valid hubloc - // and re-create it if it doesn't. + foreach($r as $rr) { + q("update hubloc set hubloc_flags = (hubloc_flags | %d) where hubloc_id = %d limit 1", + intval(HUBLOC_FLAGS_DELETED), + intval($rr['hubloc_id']) + ); + $x = q("select channel_id from channel where channel_hash = '%s' limit 1", + dbesc($rr['hubloc_hash']) + ); + if($x) { +// proc_run('php','include/notifier.php','location',$x[0]['channel_id']); +// if($interval) +// @time_sleep_until(microtime(true) + (float) $interval); + } + } } -- cgit v1.2.3 From f19d718631675d38efca1741f46bf0916af72a65 Mon Sep 17 00:00:00 2001 From: friendica Date: Tue, 30 Sep 2014 23:41:50 -0700 Subject: that's why remove_obsolete_hublocs() isn't telling anybody when it does its thing, I forgot to uncomment the bit that tells everybody after I tested it. I needed extensive testing to make sure we didn't accidentally wipe out all hublocs everywhere. Testing went fine so I just assumed it was all working as planned; but went back today to find out why I wasn't told of a recent change. --- include/hubloc.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index cdc9de4af..fded434d2 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -105,9 +105,9 @@ function remove_obsolete_hublocs() { dbesc($rr['hubloc_hash']) ); if($x) { -// proc_run('php','include/notifier.php','location',$x[0]['channel_id']); -// if($interval) -// @time_sleep_until(microtime(true) + (float) $interval); + proc_run('php','include/notifier.php','location',$x[0]['channel_id']); + if($interval) + @time_sleep_until(microtime(true) + (float) $interval); } } } -- cgit v1.2.3 From 1b0390af0659e0c5534acbee97db1ef3c1e65288 Mon Sep 17 00:00:00 2001 From: friendica Date: Mon, 13 Oct 2014 15:27:03 -0700 Subject: new function hubloc_change_primary() --- include/hubloc.php | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index fded434d2..89ec6231d 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -113,4 +113,46 @@ function remove_obsolete_hublocs() { } +function hubloc_change_primary($hubloc) { + + if(! is_array($hubloc)) + return false; + if(! $hubloc['hubloc_flags'] & HUBLOC_FLAGS_PRIMARY) + return false; + + // See if there's a local channel + + $r = q("select channel_id, channel_primary from channel where channel_hash = '%s' limit 1", + dbesc($hubloc['hubloc_hash']) + ); + if(($r) && (! $r[0]['channel_primary'])) { + q("update channel set channel_primary = 1 where channel_id = %d limit 1", + intval($r[0]['channel_id']) + ); + } + + // do we even have an xchan for this hubloc and if so is it already set as primary? + + $r = q("select * from xchan where xchan_hash = '%s' limit 1", + dbesc($hubloc['hubloc_hash']) + ); + if(! $r) + return false; + if($r[0]['xchan_addr'] === $hubloc['hubloc_addr']) + return false; + + $url = $hubloc['hubloc_url']; + $lwebbie = substr($hubloc['hubloc_addr'],0,strpos($hubloc['hubloc_addr'],'@')); + + $r = q("update xchan set xchan_addr, xchan_url = '%s', xchan_follow = '%s', xchan_connurl = '%s', where xchan_hash = '%s' limit 1", + dbesc($hubloc['hubloc_addr']), + dbesc($url . '/channel/' . $lwebbie), + dbesc($url . '/follow?f=&url=%s'), + dbesc($url . '/poco/' . $lwebbie), + dbesc($hubloc['hubloc_hash']) + ); + + return true; + +} \ No newline at end of file -- cgit v1.2.3 From 6503b911316d9573ba06b90fe45d8757488d464e Mon Sep 17 00:00:00 2001 From: friendica Date: Mon, 13 Oct 2014 16:12:32 -0700 Subject: sql --- include/hubloc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index 89ec6231d..1906833c5 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -144,7 +144,7 @@ function hubloc_change_primary($hubloc) { $url = $hubloc['hubloc_url']; $lwebbie = substr($hubloc['hubloc_addr'],0,strpos($hubloc['hubloc_addr'],'@')); - $r = q("update xchan set xchan_addr, xchan_url = '%s', xchan_follow = '%s', xchan_connurl = '%s', where xchan_hash = '%s' limit 1", + $r = q("update xchan set xchan_addr = '%s', xchan_url = '%s', xchan_follow = '%s', xchan_connurl = '%s' where xchan_hash = '%s' limit 1", dbesc($hubloc['hubloc_addr']), dbesc($url . '/channel/' . $lwebbie), dbesc($url . '/follow?f=&url=%s'), -- cgit v1.2.3 From 8a907a789f080ccae45e5b44d0bc43ed65477e07 Mon Sep 17 00:00:00 2001 From: friendica Date: Mon, 13 Oct 2014 17:59:03 -0700 Subject: more diagnostic when changing primary --- include/hubloc.php | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index 1906833c5..735aad432 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -115,10 +115,16 @@ function remove_obsolete_hublocs() { function hubloc_change_primary($hubloc) { - if(! is_array($hubloc)) + if(! is_array($hubloc)) { + logger('no hubloc'); return false; - if(! $hubloc['hubloc_flags'] & HUBLOC_FLAGS_PRIMARY) + } + if(! ($hubloc['hubloc_flags'] & HUBLOC_FLAGS_PRIMARY)) { + logger('not primary: ' . $hubloc['hubloc_url']); return false; + } + + logger('setting primary: ' . $hubloc['hubloc_url']); // See if there's a local channel @@ -136,10 +142,14 @@ function hubloc_change_primary($hubloc) { $r = q("select * from xchan where xchan_hash = '%s' limit 1", dbesc($hubloc['hubloc_hash']) ); - if(! $r) + if(! $r) { + logger('xchan not found'); return false; - if($r[0]['xchan_addr'] === $hubloc['hubloc_addr']) + } + if($r[0]['xchan_addr'] === $hubloc['hubloc_addr']) { + logger('xchan already changed'); return false; + } $url = $hubloc['hubloc_url']; $lwebbie = substr($hubloc['hubloc_addr'],0,strpos($hubloc['hubloc_addr'],'@')); @@ -151,7 +161,10 @@ function hubloc_change_primary($hubloc) { dbesc($url . '/poco/' . $lwebbie), dbesc($hubloc['hubloc_hash']) ); + if(! $r) + logger('xchan_update failed.'); + logger('primary hubloc changed.' . print_r($hubloc,true),LOGGER_DEBUG); return true; } -- cgit v1.2.3 From 0057612a9f9ecb49c2bd69e550827e2304c85204 Mon Sep 17 00:00:00 2001 From: friendica Date: Mon, 13 Oct 2014 21:38:36 -0700 Subject: figuring out how to bootstrap the change_primary procedure when all you have is inconsistent data which you think you trust. --- include/hubloc.php | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index 735aad432..04c29315a 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -113,6 +113,8 @@ function remove_obsolete_hublocs() { } +// This actually changes other structures to match the given (presumably current) hubloc primary selection + function hubloc_change_primary($hubloc) { if(! is_array($hubloc)) { -- cgit v1.2.3 From 241bb3b94059ac4edfcc3b66e907dbf493dd8b4a Mon Sep 17 00:00:00 2001 From: friendica Date: Thu, 23 Oct 2014 19:33:47 -0700 Subject: API: xchan get/create --- include/hubloc.php | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index 04c29315a..df817329f 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -170,4 +170,82 @@ function hubloc_change_primary($hubloc) { return true; } - \ No newline at end of file + + +function xchan_store($arr) { + + if(! $arr['hash']) + $arr['hash'] = $arr['guid']; + if(! $arr['hash']) + return false; + + $r = q("select * from xchan where xchan_hash = '%s' limit 1", + dbesc($arr['hash']) + ); + if($r) + return true; + + if(! $arr['network']) + $arr['network'] = 'unknown'; + if(! $arr['name']) + $arr['name'] = 'unknown'; + if(! $arr['url']) + $arr['url'] = z_root(); + if(! $arr['photo']) + $arr['photo'] = get_default_profile_photo(); + + $r = q("insert into xchan ( xchan_hash, xchan_guid, xchan_guid_sig, xchan_pubkey, xchan_addr, xchan_url, xchan_connurl, xchan_follow, xchan_connpage, xchan_name, xchan_network, xchan_instance_url, xchan_flags, xchan_name_date ) values ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s',%d,'%s') ", + dbesc($arr['hash']), + dbesc($arr['guid']), + dbesc($arr['guid_sig']), + dbesc($arr['pubkey']), + dbesc($arr['address']), + dbesc($arr['url']), + dbesc($arr['connurl']), + dbesc($arr['follow']), + dbesc($arr['connpage']), + dbesc($arr['name']), + dbesc($arr['network']), + dbesc($arr['instance_url']), + intval($arr['flags']), + dbesc(datetime_convert()) + ); + if(! $r) + return $r; + + $photos = import_profile_photo($arr['photo'],$arr['hash']); + $r = q("update xchan set xchan_photo_date = '%s', xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s' where xchan_hash = '%s' limit 1", + dbesc(datetime_convert()), + dbesc($photos[0]), + dbesc($photos[1]), + dbesc($photos[2]), + dbesc($photos[3]), + dbesc($arr['hash']) + ); + return $r; + +} + + +function xchan_fetch($arr) { + + $key = ''; + if($arr['hash']) { + $key = 'xchan_hash'; + $v = $arr['hash']; + } + elseif($arr['guid']) { + $key = 'xchan_guid'; + $v = $arr['guid']; + } + elseif($arr['address']) { + $key = 'xchan_addr'; + $v = $arr['address']; + } + + if(! $key) + return false; + + $r = q("select * from xchan where $key = '$v'"); + return $r; +} \ No newline at end of file -- cgit v1.2.3 From f524fb1f20ec3edb0cf2c486d41f6bb65a60db6a Mon Sep 17 00:00:00 2001 From: friendica Date: Thu, 23 Oct 2014 19:41:42 -0700 Subject: generalise the output format of xchan_fetch so it matches the input format --- include/hubloc.php | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index df817329f..0a1b51331 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -247,5 +247,15 @@ function xchan_fetch($arr) { return false; $r = q("select * from xchan where $key = '$v'"); - return $r; + if(! $r) + return false; + + $ret = array(); + foreach($r as $k => $v) { + if($k === 'xchan_addr') + $ret['address'] = $v; + else + $ret[str_replace('xchan_','',$k)] = $v; + } + return $ret; } \ No newline at end of file -- cgit v1.2.3 From 1a5a5c7edb8697c93f8bababbafa80245378dd7e Mon Sep 17 00:00:00 2001 From: Habeas Codice Date: Thu, 13 Nov 2014 12:21:58 -0800 Subject: PostgreSQL support initial commit There were 11 main types of changes: - UPDATE's and DELETE's sometimes had LIMIT 1 at the end of them. This is not only non-compliant but it would certainly not do what whoever wrote it thought it would. It is likely this mistake was just copied from Friendica. All of these instances, the LIMIT 1 was simply removed. - Bitwise operations (and even some non-zero int checks) erroneously rely on MySQL implicit integer-boolean conversion in the WHERE clauses. This is non-compliant (and bad programming practice to boot). Proper explicit boolean conversions were added. New queries should use proper conventions. - MySQL has a different operator for bitwise XOR than postgres. Rather than add yet another dba_ func, I converted them to "& ~" ("AND NOT") when turning off, and "|" ("OR") when turning on. There were no true toggles (XOR). New queries should refrain from using XOR when not necessary. - There are several fields which the schema has marked as NOT NULL, but the inserts don't specify them. The reason this works is because mysql totally ignores the constraint and adds an empty text default automatically. Again, non-compliant, obviously. In these cases a default of empty text was added. - Several statements rely on a non-standard MySQL feature (http://dev.mysql.com/doc/refman/5.5/en/group-by-handling.html). These queries can all be rewritten to be standards compliant. Interestingly enough, the newly rewritten standards compliant queries run a zillion times faster, even on MySQL. - A couple of function/operator name translations were needed (RAND/RANDOM, GROUP_CONCAT/STRING_AGG, UTC_NOW, REGEXP/~, ^/#) -- assist functions added in the dba_ - INTERVALs: postgres requires quotes around the value, mysql requires that there are not quotes around the value -- assist functions added in the dba_ - NULL_DATE's -- Postgres does not allow the invalid date '0000-00-00 00:00:00' (there is no such thing as year 0 or month 0 or day 0). We use '0001-01-01 00:00:00' for postgres. Conversions are handled in Zot/item packets automagically by quoting all dates with dbescdate(). - char(##) specifications in the schema creates fields with blank spaces that aren't trimmed in the code. MySQL apparently treats char(##) as varchar(##), again, non-compliant. Since postgres works better with text fields anyway, this ball of bugs was simply side-stepped by using 'text' datatype for all text fields in the postgres schema. varchar was used in a couple of places where it actually seemed appropriate (size constraint), but without rigorously vetting that all of the PHP code actually validates data, new bugs might come out from under the rug. - postgres doesn't store nul bytes and a few other non-printables in text fields, even when quoted. bytea fields were used when storing binary data (photo.data, attach.data). A new dbescbin() function was added to handle this transparently. - postgres does not support LIMIT #,# syntax. All databases support LIMIT # OFFSET # syntax. Statements were updated to be standard. These changes require corresponding changes in the coding standards. Please review those before adding any code going forward. Still on my TODO list: - remove quotes from non-reserved identifiers and make reserved identifiers use dba func for quoting - Rewrite search queries for better results (both MySQL and Postgres) --- include/hubloc.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index 0a1b51331..43187fcee 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -96,7 +96,7 @@ function remove_obsolete_hublocs() { ? intval(get_config('system','delivery_interval')) : 2 ); foreach($r as $rr) { - q("update hubloc set hubloc_flags = (hubloc_flags | %d) where hubloc_id = %d limit 1", + q("update hubloc set hubloc_flags = (hubloc_flags | %d) where hubloc_id = %d", intval(HUBLOC_FLAGS_DELETED), intval($rr['hubloc_id']) ); @@ -134,7 +134,7 @@ function hubloc_change_primary($hubloc) { dbesc($hubloc['hubloc_hash']) ); if(($r) && (! $r[0]['channel_primary'])) { - q("update channel set channel_primary = 1 where channel_id = %d limit 1", + q("update channel set channel_primary = 1 where channel_id = %d", intval($r[0]['channel_id']) ); } @@ -156,7 +156,7 @@ function hubloc_change_primary($hubloc) { $url = $hubloc['hubloc_url']; $lwebbie = substr($hubloc['hubloc_addr'],0,strpos($hubloc['hubloc_addr'],'@')); - $r = q("update xchan set xchan_addr = '%s', xchan_url = '%s', xchan_follow = '%s', xchan_connurl = '%s' where xchan_hash = '%s' limit 1", + $r = q("update xchan set xchan_addr = '%s', xchan_url = '%s', xchan_follow = '%s', xchan_connurl = '%s' where xchan_hash = '%s'", dbesc($hubloc['hubloc_addr']), dbesc($url . '/channel/' . $lwebbie), dbesc($url . '/follow?f=&url=%s'), @@ -214,7 +214,7 @@ function xchan_store($arr) { return $r; $photos = import_profile_photo($arr['photo'],$arr['hash']); - $r = q("update xchan set xchan_photo_date = '%s', xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s' where xchan_hash = '%s' limit 1", + $r = q("update xchan set xchan_photo_date = '%s', xchan_photo_l = '%s', xchan_photo_m = '%s', xchan_photo_s = '%s', xchan_photo_mimetype = '%s' where xchan_hash = '%s'", dbesc(datetime_convert()), dbesc($photos[0]), dbesc($photos[1]), -- cgit v1.2.3 From 1780ba5e7d1571d82a6428c4907d6a264217c778 Mon Sep 17 00:00:00 2001 From: friendica Date: Sun, 14 Dec 2014 22:54:27 -0800 Subject: don't remove obsolete hublocs that have no sitekey - it will remove all kinds of other hublocs it isn't supposed to (e.g. diaspora). --- include/hubloc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/hubloc.php') diff --git a/include/hubloc.php b/include/hubloc.php index 43187fcee..b5a3d47c5 100644 --- a/include/hubloc.php +++ b/include/hubloc.php @@ -33,7 +33,7 @@ function prune_hub_reinstalls() { // allow some slop period, say 3 days - just in case this is a glitch or transient occurrence // Then remove any hublocs pointing to the oldest entry. - if($d1 < $d2) { + if(($d1 < $d2) && ($x[0]['hubloc_sitekey'])) { logger('prune_hub_reinstalls: removing dead hublocs at ' . $rr['site_url']); $y = q("delete from hubloc where hubloc_sitekey = '%s'", dbesc($x[0]['hubloc_sitekey']) -- cgit v1.2.3