Commit e26a843c authored by Evan Prodromou's avatar Evan Prodromou

Offload inbox updates to a queue handler to speed up posting online

Moved much of the writing that happens when posting a notice to a new
queuehandler, distribqueuehandler. This updates tags, groups, replies
and inboxes at queue time (or at Web time, if queues are disabled).

To make this work well, I had to break up the monolithic
Notice::blowCaches() and make cache blowing happen closer to where
data is updated.

Squashed commit of the following:

commit 5257626c62750ac4ac1db0ce2b71410c5711cfa3
Author: Evan Prodromou <evan@status.net>
Date:   Mon Jan 25 14:56:41 2010 -0500

    slightly better handling of blowing tag memory cache

commit 8a22a3cdf6ec28685da129a0313e7b2a0837c9ef
Author: Evan Prodromou <evan@status.net>
Date:   Mon Jan 25 01:42:56 2010 -0500

    change 'distribute' to 'distrib' so not too long for dbqueue

commit 7a063315b0f7fad27cb6fbd2bdd74e253af83e4f
Author: Evan Prodromou <evan@status.net>
Date:   Mon Jan 25 01:39:15 2010 -0500

    change handle_notice() to handle() in distributqueuehandler

commit 1a39ccd28b9994137d7bfd21bb4f230546938e77
Author: Evan Prodromou <evan@status.net>
Date:   Mon Jan 25 16:05:25 2010 -0500

    error with queuemanager

commit e6b3bb93f305cfd2de71a6340b8aa6fb890049b7
Author: Evan Prodromou <evan@status.net>
Date:   Mon Jan 25 01:11:34 2010 -0500

    Blow memcache at different point rather than one big function for Notice class

commit 94d557cdc016187d1d0647ae1794cd94d6fb8ac8
Author: Evan Prodromou <evan@status.net>
Date:   Mon Jan 25 00:48:44 2010 -0500

    Blow memcache at different point rather than one big function for Notice class

commit 1c781dd08c88a35dafc5c01230b4872fd6b95182
Author: Evan Prodromou <evan@status.net>
Date:   Wed Jan 20 08:54:18 2010 -0500

    move broadcasting and distributing to new queuehandler

commit da3e46d26b84e4f028f34a13fd2ee373e4c1b954
Author: Evan Prodromou <evan@status.net>
Date:   Wed Jan 20 08:53:12 2010 -0500

    Move distribution of notices to new distribute queue handler
parent f3beed68
......@@ -112,7 +112,7 @@ class ApiStatusesRetweetAction extends ApiAuthAction
$repeat = $this->original->repeat($this->user->id, $this->source);
common_broadcast_notice($repeat);
$this->showNotice($repeat);
}
......
......@@ -250,7 +250,7 @@ class ApiStatusesUpdateAction extends ApiAuthAction
$upload->attachToNotice($this->notice);
}
common_broadcast_notice($this->notice);
}
$this->showNotice();
......
......@@ -201,7 +201,7 @@ class NewnoticeAction extends Action
$upload->attachToNotice($notice);
}
common_broadcast_notice($notice);
if ($this->boolean('ajax')) {
header('Content-Type: text/xml;charset=utf-8');
......
......@@ -106,7 +106,7 @@ class RepeatAction extends Action
{
$repeat = $this->notice->repeat($this->user->id, 'web');
common_broadcast_notice($repeat);
if ($this->boolean('ajax')) {
$this->startHTML('text/xml;charset=utf-8');
......
......@@ -120,11 +120,7 @@ class Inbox extends Memcached_DataObject
$notice_id, $user_id));
if ($result) {
$c = self::memcache();
if (!empty($c)) {
$c->delete(self::cacheKey('inbox', 'user_id', $user_id));
}
self::blow('inbox:user_id:%d', $user_id);
}
return $result;
......
......@@ -425,4 +425,23 @@ class Memcached_DataObject extends DB_DataObject
return $dsn;
}
static function blow()
{
$c = self::memcache();
if (empty($c)) {
return false;
}
$args = func_get_args();
$format = array_shift($args);
$keyPart = vsprintf($format, $args);
$cacheKey = common_cache_key($keyPart);
return $c->delete($cacheKey);
}
}
This diff is collapsed.
......@@ -86,13 +86,9 @@ class Notice_tag extends Memcached_DataObject
function blowCache($blowLast=false)
{
$cache = common_memcache();
if ($cache) {
$idkey = common_cache_key('notice_tag:notice_ids:' . common_keyize($this->tag));
$cache->delete($idkey);
if ($blowLast) {
$cache->delete($idkey.';last');
}
self::blow('notice_tag:notice_ids:%s', common_keyize($this->tag));
if ($blowLast) {
self::blow('notice_tag:notice_ids:%s;last', common_keyize($this->tag));
}
}
......
......@@ -383,7 +383,7 @@ class User extends Memcached_DataObject
common_config('site', 'name'),
$user->nickname),
'system');
common_broadcast_notice($notice);
}
}
......
......@@ -422,7 +422,7 @@ class RepeatCommand extends Command
$repeat = $notice->repeat($this->user->id, $channel->source);
if ($repeat) {
common_broadcast_notice($repeat);
$channel->output($this->user, sprintf(_('Notice from %s repeated'), $recipient->nickname));
} else {
$channel->error($this->user, _('Error repeating notice.'));
......@@ -492,7 +492,7 @@ class ReplyCommand extends Command
} else {
$channel->error($this->user, _('Error saving notice.'));
}
common_broadcast_notice($notice);
}
}
......
<?php
/*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2008, 2009, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
/**
* Base class for queue handlers.
*
* As extensions of the Daemon class, each queue handler has the ability
* to launch itself in the background, at which point it'll pass control
* to the configured QueueManager class to poll for updates.
*
* Subclasses must override at least the following methods:
* - transport
* - handle_notice
*/
class DistribQueueHandler
{
/**
* Return transport keyword which identifies items this queue handler
* services; must be defined for all subclasses.
*
* Must be 8 characters or less to fit in the queue_item database.
* ex "email", "jabber", "sms", "irc", ...
*
* @return string
*/
function transport()
{
return 'distrib';
}
/**
* Here's the meat of your queue handler -- you're handed a Notice
* object, which you may do as you will with.
*
* If this function indicates failure, a warning will be logged
* and the item is placed back in the queue to be re-run.
*
* @param Notice $notice
* @return boolean true on success, false on failure
*/
function handle($notice)
{
// XXX: do we need to change this for remote users?
$notice->saveTags();
$groups = $notice->saveGroups();
$recipients = $notice->saveReplies();
$notice->addToInboxes($groups, $recipients);
$notice->saveUrls();
Event::handle('EndNoticeSave', array($notice));
// Enqueue for other handlers
common_enqueue_notice($notice);
return true;
}
}
......@@ -160,7 +160,7 @@ class MailHandler
foreach($mediafiles as $mf){
$mf->attachToNotice($notice);
}
common_broadcast_notice($notice);
$this->log(LOG_INFO,
'Added notice ' . $notice->id . ' from user ' . $user->nickname);
return true;
......
......@@ -362,7 +362,7 @@ class StatusNetOAuthDataStore extends OAuthDataStore
array('is_local' => Notice::REMOTE_OMB,
'uri' => $omb_notice->getIdentifierURI()));
common_broadcast_notice($notice, true);
}
/**
......
......@@ -217,6 +217,7 @@ abstract class QueueManager extends IoManager
$this->connect('plugin', 'PluginQueueHandler');
$this->connect('omb', 'OmbQueueHandler');
$this->connect('ping', 'PingQueueHandler');
$this->connect('distrib', 'DistribQueueHandler');
if (common_config('sms', 'enabled')) {
$this->connect('sms', 'SmsQueueHandler');
}
......@@ -224,7 +225,7 @@ abstract class QueueManager extends IoManager
// XMPP output handlers...
$this->connect('jabber', 'JabberQueueHandler');
$this->connect('public', 'PublicQueueHandler');
// @fixme this should get an actual queue
//$this->connect('confirm', 'XmppConfirmHandler');
......
......@@ -987,7 +987,7 @@ function common_redirect($url, $code=307)
function common_broadcast_notice($notice, $remote=false)
{
return common_enqueue_notice($notice);
// DO NOTHING!
}
// Stick the notice on the queue
......
......@@ -397,7 +397,7 @@ class FacebookAction extends Action
return;
}
common_broadcast_notice($notice);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment