We are no longer offering accounts on this server. Consider https://gitlab.freedesktop.org/ as a place to host projects.

Memcached_DataObject.php 19.7 KB
Newer Older
1 2
<?php
/*
3
 * StatusNet - the distributed open-source microblogging tool
4
 * Copyright (C) 2008, 2009, StatusNet, Inc.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

20
if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); }
21

22
class Memcached_DataObject extends Safe_DataObject
23
{
24 25 26 27 28 29 30 31 32
    /**
     * Wrapper for DB_DataObject's static lookup using memcached
     * as backing instead of an in-process cache array.
     *
     * @param string $cls classname of object type to load
     * @param mixed $k key field name, or value for primary key
     * @param mixed $v key field value, or leave out for primary key lookup
     * @return mixed Memcached_DataObject subtype or false
     */
33 34
    function &staticGet($cls, $k, $v=null)
    {
35 36
        if (is_null($v)) {
            $v = $k;
37
            // XXX: HACK!
38 39 40 41 42
            $i = new $cls;
            $keys = $i->keys();
            $k = $keys[0];
            unset($i);
        }
43
        $i = Memcached_DataObject::getcached($cls, $k, $v);
44
        if ($i === false) { // false == cache miss
45 46
            $i = DB_DataObject::factory($cls);
            if (empty($i)) {
47 48
                $i = false;
                return $i;
49 50 51
            }
            $result = $i->get($k, $v);
            if ($result) {
52
                // Hit!
53
                $i->encache();
54
            } else {
55 56 57 58 59 60
                // save the fact that no such row exists
                $c = self::memcache();
                if (!empty($c)) {
                    $ck = self::cachekey($cls, $k, $v);
                    $c->set($ck, null);
                }
61
                $i = false;
62 63
            }
        }
64
        return $i;
65
    }
66

67 68 69 70
    /**
     * @fixme Should this return false on lookup fail to match staticGet?
     */
    function pkeyGet($cls, $kv)
71
    {
72
        $i = Memcached_DataObject::multicache($cls, $kv);
73
        if ($i !== false) { // false == cache miss
74 75
            return $i;
        } else {
76
            $i = DB_DataObject::factory($cls);
77
            if (empty($i) || PEAR::isError($i)) {
78 79
                return false;
            }
80 81 82 83 84 85
            foreach ($kv as $k => $v) {
                $i->$k = $v;
            }
            if ($i->find(true)) {
                $i->encache();
            } else {
Evan Prodromou's avatar
Evan Prodromou committed
86
                $i = null;
87 88 89 90 91
                $c = self::memcache();
                if (!empty($c)) {
                    $ck = self::multicacheKey($cls, $kv);
                    $c->set($ck, null);
                }
92
            }
93
            return $i;
94 95
        }
    }
96

97 98
    function insert()
    {
99
        $result = parent::insert();
100
        if ($result) {
101
            $this->fixupTimestamps();
102 103
            $this->encache(); // in case of cached negative lookups
        }
104 105
        return $result;
    }
106

107 108
    function update($orig=null)
    {
109 110 111 112 113
        if (is_object($orig) && $orig instanceof Memcached_DataObject) {
            $orig->decache(); # might be different keys
        }
        $result = parent::update($orig);
        if ($result) {
114
            $this->fixupTimestamps();
115 116 117 118
            $this->encache();
        }
        return $result;
    }
119

120 121
    function delete()
    {
122 123 124
        $this->decache(); # while we still have the values!
        return parent::delete();
    }
125

126
    static function memcache() {
127
        return Cache::instance();
128
    }
129

130
    static function cacheKey($cls, $k, $v) {
131
        if (is_object($cls) || is_object($k) || (is_object($v) && !($v instanceof DB_DataObject_Cast))) {
132 133 134 135
            $e = new Exception();
            common_log(LOG_ERR, __METHOD__ . ' object in param: ' .
                str_replace("\n", " ", $e->getTraceAsString()));
        }
136
        $vstr = self::valueString($v);
137
        return Cache::key(strtolower($cls).':'.$k.':'.$vstr);
138
    }
139

140
    static function getcached($cls, $k, $v) {
141
        $c = Memcached_DataObject::memcache();
142 143 144
        if (!$c) {
            return false;
        } else {
145 146 147
            $obj = $c->get(Memcached_DataObject::cacheKey($cls, $k, $v));
            if (0 == strcasecmp($cls, 'User')) {
                // Special case for User
148
                if (is_object($obj) && is_object($obj->id)) {
149 150 151 152 153 154
                    common_log(LOG_ERR, "User " . $obj->nickname . " was cached with User as ID; deleting");
                    $c->delete(Memcached_DataObject::cacheKey($cls, $k, $v));
                    return false;
                }
            }
            return $obj;
155 156
        }
    }
157

158 159
    function keyTypes()
    {
160 161 162 163 164 165 166 167 168 169 170
        // ini-based classes return number-indexed arrays. handbuilt
        // classes return column => keytype. Make this uniform.

        $keys = $this->keys();

        $keyskeys = array_keys($keys);

        if (is_string($keyskeys[0])) {
            return $keys;
        }

171
        global $_DB_DATAOBJECT;
172
        if (!isset($_DB_DATAOBJECT['INI'][$this->_database][$this->__table."__keys"])) {
173
            $this->databaseStructure();
174 175

        }
176 177
        return $_DB_DATAOBJECT['INI'][$this->_database][$this->__table."__keys"];
    }
178

179 180
    function encache()
    {
181
        $c = $this->memcache();
182

183 184
        if (!$c) {
            return false;
185 186 187 188 189 190
        } else if ($this->tableName() == 'user' && is_object($this->id)) {
            // Special case for User bug
            $e = new Exception();
            common_log(LOG_ERR, __METHOD__ . ' caching user with User object as ID ' .
                       str_replace("\n", " ", $e->getTraceAsString()));
            return false;
191
        } else {
192
            $keys = $this->_allCacheKeys();
193

194 195 196
            foreach ($keys as $key) {
                $c->set($key, $this);
            }
197 198
        }
    }
199

200 201
    function decache()
    {
202
        $c = $this->memcache();
203

204 205
        if (!$c) {
            return false;
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
        }

        $keys = $this->_allCacheKeys();

        foreach ($keys as $key) {
            $c->delete($key, $this);
        }
    }

    function _allCacheKeys()
    {
        $ckeys = array();

        $types = $this->keyTypes();
        ksort($types);

        $pkey = array();
        $pval = array();

        foreach ($types as $key => $type) {

            assert(!empty($key));

            if ($type == 'U') {
                if (empty($this->$key)) {
                    continue;
232
                }
233
                $ckeys[] = $this->cacheKey($this->tableName(), $key, self::valueString($this->$key));
234 235
            } else if ($type == 'K' || $type == 'N') {
                $pkey[] = $key;
236
                $pval[] = self::valueString($this->$key);
237
            } else {
238
                // Low level exception. No need for i18n as discussed with Brion.
239
                throw new Exception("Unknown key type $key => $type for " . $this->tableName());
240 241
            }
        }
242 243 244 245 246 247 248 249 250 251

        assert(count($pkey) > 0);

        // XXX: should work for both compound and scalar pkeys
        $pvals = implode(',', $pval);
        $pkeys = implode(',', $pkey);

        $ckeys[] = $this->cacheKey($this->tableName(), $pkeys, $pvals);

        return $ckeys;
252
    }
253

254 255
    function multicache($cls, $kv)
    {
256
        ksort($kv);
257
        $c = self::memcache();
258 259 260
        if (!$c) {
            return false;
        } else {
261
            return $c->get(self::multicacheKey($cls, $kv));
262 263
        }
    }
millette's avatar
millette committed
264

265 266 267 268 269 270 271 272
    static function multicacheKey($cls, $kv)
    {
        ksort($kv);
        $pkeys = implode(',', array_keys($kv));
        $pvals = implode(',', array_values($kv));
        return self::cacheKey($cls, $pkeys, $pvals);
    }

273 274
    function getSearchEngine($table)
    {
millette's avatar
millette committed
275
        require_once INSTALLDIR.'/lib/search_engines.php';
276 277 278 279 280 281 282 283

        if (Event::handle('GetSearchEngine', array($this, $table, &$search_engine))) {
            if ('mysql' === common_config('db', 'type')) {
                $type = common_config('search', 'type');
                if ($type == 'like') {
                    $search_engine = new MySQLLikeSearch($this, $table);
                } else if ($type == 'fulltext') {
                    $search_engine = new MySQLSearch($this, $table);
284
                } else {
285 286
                    // Low level exception. No need for i18n as discussed with Brion.
                    throw new ServerException('Unknown search type: ' . $type);
millette's avatar
millette committed
287
                }
288 289
            } else {
                $search_engine = new PGSearch($this, $table);
290
            }
millette's avatar
millette committed
291
        }
292

millette's avatar
millette committed
293 294
        return $search_engine;
    }
295 296 297

    static function cachedQuery($cls, $qry, $expiry=3600)
    {
298
        $c = Memcached_DataObject::memcache();
299 300 301
        if (!$c) {
            $inst = new $cls();
            $inst->query($qry);
302
            return $inst;
303
        }
304
        $key_part = Cache::keyize($cls).':'.md5($qry);
305
        $ckey = Cache::key($key_part);
306
        $stored = $c->get($ckey);
307 308

        if ($stored !== false) {
309 310 311 312
            return new ArrayWrapper($stored);
        }

        $inst = new $cls();
313
        $inst->query($qry);
314 315 316 317 318
        $cached = array();
        while ($inst->fetch()) {
            $cached[] = clone($inst);
        }
        $inst->free();
319
        $c->set($ckey, $cached, Cache::COMPRESSED, $expiry);
320
        return new ArrayWrapper($cached);
321
    }
322

323
    /**
324
     * sends query to database - this is the private one that must work
325 326 327 328 329 330 331 332 333 334
     *   - internal functions use this rather than $this->query()
     *
     * Overridden to do logging.
     *
     * @param  string  $string
     * @access private
     * @return mixed none or PEAR_Error
     */
    function _query($string)
    {
335 336 337 338
        if (common_config('db', 'annotate_queries')) {
            $string = $this->annotateQuery($string);
        }

339
        $start = microtime(true);
340
        $fail = false;
Brion Vibber's avatar
Brion Vibber committed
341 342
        $result = null;
        if (Event::handle('StartDBQuery', array($this, $string, &$result))) {
343
            common_perf_counter('query', $string);
Brion Vibber's avatar
Brion Vibber committed
344 345 346 347 348
            try {
                $result = parent::_query($string);
            } catch (Exception $e) {
                $fail = $e;
            }
Brion Vibber's avatar
Brion Vibber committed
349
            Event::handle('EndDBQuery', array($this, $string, &$result));
350
        }
351 352 353 354 355
        $delta = microtime(true) - $start;

        $limit = common_config('db', 'log_slow_queries');
        if (($limit > 0 && $delta >= $limit) || common_config('db', 'log_queries')) {
            $clean = $this->sanitizeQuery($string);
356 357 358 359 360 361 362 363 364 365
            if ($fail) {
                $msg = sprintf("FAILED DB query (%0.3fs): %s - %s", $delta, $fail->getMessage(), $clean);
            } else {
                $msg = sprintf("DB query (%0.3fs): %s", $delta, $clean);
            }
            common_log(LOG_DEBUG, $msg);
        }

        if ($fail) {
            throw $fail;
366 367 368 369
        }
        return $result;
    }

370 371 372 373 374
    /**
     * Find the first caller in the stack trace that's not a
     * low-level database function and add a comment to the
     * query string. This should then be visible in process lists
     * and slow query logs, to help identify problem areas.
375
     *
376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
     * Also marks whether this was a web GET/POST or which daemon
     * was running it.
     *
     * @param string $string SQL query string
     * @return string SQL query string, with a comment in it
     */
    function annotateQuery($string)
    {
        $ignore = array('annotateQuery',
                        '_query',
                        'query',
                        'get',
                        'insert',
                        'delete',
                        'update',
                        'find');
        $ignoreStatic = array('staticGet',
                              'pkeyGet',
                              'cachedQuery');
        $here = get_class($this); // if we get confused
        $bt = debug_backtrace();

        // Find the first caller that's not us?
        foreach ($bt as $frame) {
            $func = $frame['function'];
            if (isset($frame['type']) && $frame['type'] == '::') {
                if (in_array($func, $ignoreStatic)) {
                    continue;
                }
                $here = $frame['class'] . '::' . $func;
                break;
            } else if (isset($frame['type']) && $frame['type'] == '->') {
                if ($frame['object'] === $this && in_array($func, $ignore)) {
                    continue;
                }
                if (in_array($func, $ignoreStatic)) {
                    continue; // @fixme this shouldn't be needed?
                }
                $here = get_class($frame['object']) . '->' . $func;
                break;
            }
            $here = $func;
            break;
        }

        if (php_sapi_name() == 'cli') {
            $context = basename($_SERVER['PHP_SELF']);
        } else {
            $context = $_SERVER['REQUEST_METHOD'];
        }

        // Slip the comment in after the first command,
        // or DB_DataObject gets confused about handling inserts and such.
        $parts = explode(' ', $string, 2);
        $parts[0] .= " /* $context $here */";
        return implode(' ', $parts);
    }

434 435 436 437 438 439 440 441 442
    // Sanitize a query for logging
    // @fixme don't trim spaces in string literals
    function sanitizeQuery($string)
    {
        $string = preg_replace('/\s+/', ' ', $string);
        $string = trim($string);
        return $string;
    }

443 444 445 446 447 448
    // We overload so that 'SET NAMES "utf8"' is called for
    // each connection

    function _connect()
    {
        global $_DB_DATAOBJECT;
449 450 451 452 453 454 455 456 457 458

        $sum = $this->_getDbDsnMD5();

        if (!empty($_DB_DATAOBJECT['CONNECTIONS'][$sum]) &&
            !PEAR::isError($_DB_DATAOBJECT['CONNECTIONS'][$sum])) {
            $exists = true;
        } else {
            $exists = false;
       }

459 460 461 462 463 464 465 466 467 468 469 470 471 472
        // @fixme horrible evil hack!
        //
        // In multisite configuration we don't want to keep around a separate
        // connection for every database; we could end up with thousands of
        // connections open per thread. In an ideal world we might keep
        // a connection per server and select different databases, but that'd
        // be reliant on having the same db username/pass as well.
        //
        // MySQL connections are cheap enough we're going to try just
        // closing out the old connection and reopening when we encounter
        // a new DSN.
        //
        // WARNING WARNING if we end up actually using multiple DBs at a time
        // we'll need some fancier logic here.
473
        if (!$exists && !empty($_DB_DATAOBJECT['CONNECTIONS']) && php_sapi_name() == 'cli') {
474 475 476 477 478 479 480
            foreach ($_DB_DATAOBJECT['CONNECTIONS'] as $index => $conn) {
                if (!empty($conn)) {
                    $conn->disconnect();
                }
                unset($_DB_DATAOBJECT['CONNECTIONS'][$index]);
            }
        }
481

482
        $result = parent::_connect();
483 484

        if ($result && !$exists) {
485
            $DB = &$_DB_DATAOBJECT['CONNECTIONS'][$this->_database_dsn_md5];
486 487 488
            if (common_config('db', 'type') == 'mysql' &&
                common_config('db', 'utf8')) {
                $conn = $DB->connection;
489 490 491 492 493 494
                if (!empty($conn)) {
                    if ($DB instanceof DB_mysqli) {
                        mysqli_set_charset($conn, 'utf8');
                    } else if ($DB instanceof DB_mysql) {
                        mysql_set_charset('utf8', $conn);
                    }
495
                }
496
            }
497 498 499 500
            // Needed to make timestamp values usefully comparable.
            if (common_config('db', 'type') == 'mysql') {
                parent::_query("set time_zone='+0:00'");
            }
501
        }
502

503 504
        return $result;
    }
505

506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553
    // XXX: largely cadged from DB_DataObject

    function _getDbDsnMD5()
    {
        if ($this->_database_dsn_md5) {
            return $this->_database_dsn_md5;
        }

        $dsn = $this->_getDbDsn();

        if (is_string($dsn)) {
            $sum = md5($dsn);
        } else {
            /// support array based dsn's
            $sum = md5(serialize($dsn));
        }

        return $sum;
    }

    function _getDbDsn()
    {
        global $_DB_DATAOBJECT;

        if (empty($_DB_DATAOBJECT['CONFIG'])) {
            DB_DataObject::_loadConfig();
        }

        $options = &$_DB_DATAOBJECT['CONFIG'];

        // if the databse dsn dis defined in the object..

        $dsn = isset($this->_database_dsn) ? $this->_database_dsn : null;

        if (!$dsn) {

            if (!$this->_database) {
                $this->_database = isset($options["table_{$this->__table}"]) ? $options["table_{$this->__table}"] : null;
            }

            if ($this->_database && !empty($options["database_{$this->_database}"]))  {
                $dsn = $options["database_{$this->_database}"];
            } else if (!empty($options['database'])) {
                $dsn = $options['database'];
            }
        }

        if (!$dsn) {
554
            // TRANS: Exception thrown when database name or Data Source Name could not be found.
555
            throw new Exception(_("No database name or DSN found anywhere."));
556 557 558 559
        }

        return $dsn;
    }
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574

    static function blow()
    {
        $c = self::memcache();

        if (empty($c)) {
            return false;
        }

        $args = func_get_args();

        $format = array_shift($args);

        $keyPart = vsprintf($format, $args);

575
        $cacheKey = Cache::key($keyPart);
576 577 578

        return $c->delete($cacheKey);
    }
579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594

    function fixupTimestamps()
    {
        // Fake up timestamp columns
        $columns = $this->table();
        foreach ($columns as $name => $type) {
            if ($type & DB_DATAOBJECT_MYSQLTIMESTAMP) {
                $this->$name = common_sql_now();
            }
        }
    }

    function debugDump()
    {
        common_debug("debugDump: " . common_log_objstring($this));
    }
595 596 597

    function raiseError($message, $type = null, $behaviour = null)
    {
598
        $id = get_class($this);
599
        if (!empty($this->id)) {
600 601
            $id .= ':' . $this->id;
        }
602 603 604
        if ($message instanceof PEAR_Error) {
            $message = $message->getMessage();
        }
605
        // Low level exception. No need for i18n as discussed with Brion.
606
        throw new ServerException("[$id] DB_DataObject error [$type]: $message");
607
    }
608 609 610 611 612 613 614 615 616

    static function cacheGet($keyPart)
    {
        $c = self::memcache();

        if (empty($c)) {
            return false;
        }

617
        $cacheKey = Cache::key($keyPart);
618 619 620 621

        return $c->get($cacheKey);
    }

622
    static function cacheSet($keyPart, $value, $flag=null, $expiry=null)
623 624 625 626 627 628 629
    {
        $c = self::memcache();

        if (empty($c)) {
            return false;
        }

630
        $cacheKey = Cache::key($keyPart);
631

632
        return $c->set($cacheKey, $value, $flag, $expiry);
633
    }
634 635 636 637 638 639 640 641 642 643 644 645 646 647

    static function valueString($v)
    {
        $vstr = null;
        if (is_object($v) && $v instanceof DB_DataObject_Cast) {
            switch ($v->type) {
            case 'date':
                $vstr = $v->year . '-' . $v->month . '-' . $v->day;
                break;
            case 'blob':
            case 'string':
            case 'sql':
            case 'datetime':
            case 'time':
648
                // Low level exception. No need for i18n as discussed with Brion.
649 650 651
                throw new ServerException("Unhandled DB_DataObject_Cast type passed as cacheKey value: '$v->type'");
                break;
            default:
652
                // Low level exception. No need for i18n as discussed with Brion.
653 654 655 656 657 658 659 660
                throw new ServerException("Unknown DB_DataObject_Cast type passed as cacheKey value: '$v->type'");
                break;
            }
        } else {
            $vstr = strval($v);
        }
        return $vstr;
    }
661
}