We are no longer offering accounts on this server. Consider https://gitlab.freedesktop.org/ as a place to host projects.

queuemanager.php 11.3 KB
Newer Older
Evan Prodromou's avatar
Evan Prodromou committed
1 2
<?php
/**
3
 * StatusNet, the distributed open-source microblogging tool
Evan Prodromou's avatar
Evan Prodromou committed
4
 *
5
 * Abstract class for i/o managers
Evan Prodromou's avatar
Evan Prodromou committed
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *
 * PHP version 5
 *
 * LICENCE: This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * @category  QueueManager
23
 * @package   StatusNet
24 25
 * @author    Evan Prodromou <evan@status.net>
 * @author    Sarven Capadisli <csarven@status.net>
26 27
 * @author    Brion Vibber <brion@status.net>
 * @copyright 2009-2010 StatusNet, Inc.
Evan Prodromou's avatar
Evan Prodromou committed
28
 * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
29
 * @link      http://status.net/
Evan Prodromou's avatar
Evan Prodromou committed
30 31
 */

32 33 34 35 36 37 38
/**
 * Completed child classes must implement the enqueue() method.
 *
 * For background processing, classes should implement either socket-based
 * input (handleInput(), getSockets()) or idle-loop polling (idle()).
 */
abstract class QueueManager extends IoManager
Evan Prodromou's avatar
Evan Prodromou committed
39 40 41
{
    static $qm = null;

42 43 44 45
    protected $master = null;
    protected $handlers = array();
    protected $groups = array();
    protected $activeGroups = array();
46

47 48 49 50 51 52 53 54 55 56
    /**
     * Factory function to pull the appropriate QueueManager object
     * for this site's configuration. It can then be used to queue
     * events for later processing or to spawn a processing loop.
     *
     * Plugins can add to the built-in types by hooking StartNewQueueManager.
     *
     * @return QueueManager
     */
    public static function get()
Evan Prodromou's avatar
Evan Prodromou committed
57 58 59
    {
        if (empty(self::$qm)) {

60
            if (Event::handle('StartNewQueueManager', array(&self::$qm))) {
Evan Prodromou's avatar
Evan Prodromou committed
61

62
                $enabled = common_config('queue', 'enabled');
63
                $type = common_config('queue', 'subsystem');
64 65 66

                if (!$enabled) {
                    // does everything immediately
67 68 69 70 71 72 73 74 75 76 77 78
                    self::$qm = new UnQueueManager();
                } else {
                    switch ($type) {
                     case 'db':
                        self::$qm = new DBQueueManager();
                        break;
                     case 'stomp':
                        self::$qm = new StompQueueManager();
                        break;
                     default:
                        throw new ServerException("No queue manager class for type '$type'");
                    }
Evan Prodromou's avatar
Evan Prodromou committed
79 80 81
                }
            }
        }
82 83

        return self::$qm;
Evan Prodromou's avatar
Evan Prodromou committed
84 85
    }

86 87 88 89 90 91 92 93 94 95 96 97 98 99
    /**
     * @fixme wouldn't necessarily work with other class types.
     * Better to change the interface...?
     */
    public static function multiSite()
    {
        if (common_config('queue', 'subsystem') == 'stomp') {
            return IoManager::INSTANCE_PER_PROCESS;
        } else {
            return IoManager::SINGLE_ONLY;
        }
    }

    function __construct()
Evan Prodromou's avatar
Evan Prodromou committed
100
    {
101
        $this->initialize();
Evan Prodromou's avatar
Evan Prodromou committed
102 103
    }

104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
    /**
     * Optional; ping any running queue handler daemons with a notification
     * such as announcing a new site to handle or requesting clean shutdown.
     * This avoids having to restart all the daemons manually to update configs
     * and such.
     *
     * Called from scripts/queuectl.php controller utility.
     *
     * @param string $event event key
     * @param string $param optional parameter to append to key
     * @return boolean success
     */
    public function sendControlSignal($event, $param='')
    {
        throw new Exception(get_class($this) . " does not support control signals.");
    }

121 122 123 124 125 126 127 128 129 130 131 132 133
    /**
     * Store an object (usually/always a Notice) into the given queue
     * for later processing. No guarantee is made on when it will be
     * processed; it could be immediately or at some unspecified point
     * in the future.
     *
     * Must be implemented by any queue manager.
     *
     * @param Notice $object
     * @param string $queue
     */
    abstract function enqueue($object, $queue);

134 135 136 137 138 139 140 141 142 143 144 145
    /**
     * Build a representation for an object for logging
     * @param mixed
     * @return string
     */
    function logrep($object) {
        if (is_object($object)) {
            $class = get_class($object);
            if (isset($object->id)) {
                return "$class $object->id";
            }
            return $class;
146
        } elseif (is_string($object)) {
147 148 149 150 151 152
            $len = strlen($object);
            $fragment = mb_substr($object, 0, 32);
            if (mb_strlen($object) > 32) {
                $fragment .= '...';
            }
            return "string '$fragment' ($len bytes)";
153 154 155
        } elseif (is_array($object)) {
            return 'array with ' . count($object) .
                   ' elements (keys:[' .  implode(',', array_keys($object)) . '])';
156 157 158 159 160 161 162
        }
        return strval($object);
    }

    /**
     * Encode an object for queued storage.
     *
163
     * @param mixed $item
164 165
     * @return string
     */
166
    protected function encode($item)
167
    {
168
        return serialize($item);
169 170 171 172
    }

    /**
     * Decode an object from queued storage.
173
     * Accepts notice reference entries and serialized items.
174 175 176 177 178 179
     *
     * @param string
     * @return mixed
     */
    protected function decode($frame)
    {
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
        $object = unserialize($frame);

        // If it is a string, we really store a JSON object in there
        if (is_string($object)) {
            $json = json_decode($object);
            if ($json === null) {
                throw new Exception('Bad frame in queue item');
            }

            // The JSON object has a type parameter which contains the class
            if (empty($json->type)) {
                throw new Exception('Type not specified for queue item');
            }
            if (!is_a($json->type, 'Managed_DataObject', true)) {
                throw new Exception('Managed_DataObject class does not exist for queue item');
            }

            // And each of these types should have a unique id (or uri)
            if (isset($json->id) && !empty($json->id)) {
                $object = call_user_func(array($json->type, 'getKV'), 'id', $json->id);
            } elseif (isset($json->uri) && !empty($json->uri)) {
                $object = call_user_func(array($json->type, 'getKV'), 'uri', $json->uri);
            }

            // But if no object was found, there's nothing we can handle
            if (!$object instanceof Managed_DataObject) {
                throw new Exception('Queue item frame referenced a non-existant object');
            }
        }

        // If the frame was not a string, it's either an array or an object.

        return $object;
213 214
    }

215 216 217 218 219 220 221
    /**
     * Instantiate the appropriate QueueHandler class for the given queue.
     *
     * @param string $queue
     * @return mixed QueueHandler or null
     */
    function getHandler($queue)
Evan Prodromou's avatar
Evan Prodromou committed
222
    {
223 224
        if (isset($this->handlers[$queue])) {
            $class = $this->handlers[$queue];
225 226 227
            if(is_object($class)) {
                return $class;
            } else if (class_exists($class)) {
228 229
                return new $class();
            } else {
230
                $this->_log(LOG_ERR, "Nonexistent handler class '$class' for queue '$queue'");
231 232
            }
        } else {
233
            $this->_log(LOG_ERR, "Requested handler for unkown queue '$queue'");
234 235 236 237 238
        }
        return null;
    }

    /**
239
     * Get a list of registered queue transport names to be used
240
     * for listening in this daemon.
241 242 243
     *
     * @return array of strings
     */
244
    function activeQueues()
245
    {
246 247 248 249 250 251 252 253
        $queues = array();
        foreach ($this->activeGroups as $group) {
            if (isset($this->groups[$group])) {
                $queues = array_merge($queues, $this->groups[$group]);
            }
        }

        return array_keys($queues);
254 255 256
    }

    /**
257
     * Initialize the list of queue handlers for the current site.
258 259 260 261 262 263
     *
     * @event StartInitializeQueueManager
     * @event EndInitializeQueueManager
     */
    function initialize()
    {
264 265 266 267
        $this->handlers = array();
        $this->groups = array();
        $this->groupsByTransport = array();

268
        if (Event::handle('StartInitializeQueueManager', array($this))) {
269
            $this->connect('distrib', 'DistribQueueHandler');
270 271 272
            $this->connect('ping', 'PingQueueHandler');
            if (common_config('sms', 'enabled')) {
                $this->connect('sms', 'SmsQueueHandler');
273 274
            }

275 276
            // Background user management tasks...
            $this->connect('deluser', 'DelUserQueueHandler');
277 278
            $this->connect('feedimp', 'FeedImporter');
            $this->connect('actimp', 'ActivityImporter');
279 280
            $this->connect('acctmove', 'AccountMover');
            $this->connect('actmove', 'ActivityMover');
281

282 283
            // For compat with old plugins not registering their own handlers.
            $this->connect('plugin', 'PluginQueueHandler');
284
        }
285
        Event::handle('EndInitializeQueueManager', array($this));
286 287 288 289 290 291 292
    }

    /**
     * Register a queue transport name and handler class for your plugin.
     * Only registered transports will be reliably picked up!
     *
     * @param string $transport
293
     * @param string $class class name or object instance
294
     * @param string $group
295
     */
296
    public function connect($transport, $class, $group='main')
297 298
    {
        $this->handlers[$transport] = $class;
299
        $this->groups[$group][$transport] = $class;
300
        $this->groupsByTransport[$transport] = $group;
301 302 303
    }

    /**
304 305
     * Set the active group which will be used for listening.
     * @param string $group
306
     */
307
    function setActiveGroup($group)
308
    {
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
        $this->activeGroups = array($group);
    }

    /**
     * Set the active group(s) which will be used for listening.
     * @param array $groups
     */
    function setActiveGroups($groups)
    {
        $this->activeGroups = $groups;
    }

    /**
     * @return string queue group for this queue
     */
    function queueGroup($queue)
    {
        if (isset($this->groupsByTransport[$queue])) {
            return $this->groupsByTransport[$queue];
        } else {
            throw new Exception("Requested group for unregistered transport $queue");
330
        }
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352
    }

    /**
     * Send a statistic ping to the queue monitoring system,
     * optionally with a per-queue id.
     *
     * @param string $key
     * @param string $queue
     */
    function stats($key, $queue=false)
    {
        $owners = array();
        if ($queue) {
            $owners[] = "queue:$queue";
            $owners[] = "site:" . common_config('site', 'server');
        }
        if (isset($this->master)) {
            $this->master->stats($key, $owners);
        } else {
            $monitor = new QueueMonitor();
            $monitor->stats($key, $owners);
        }
353
    }
354 355 356 357 358 359 360 361 362 363 364

    protected function _log($level, $msg)
    {
        $class = get_class($this);
        if ($this->activeGroups) {
            $groups = ' (' . implode(',', $this->activeGroups) . ')';
        } else {
            $groups = '';
        }
        common_log($level, "$class$groups: $msg");
    }
Evan Prodromou's avatar
Evan Prodromou committed
365
}