git.gnu.io has moved to IP address 209.51.188.249 -- please double check where you are logging in.

twitterimport.php 22.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
<?php
/**
 * StatusNet, the distributed open-source microblogging tool
 *
 * PHP version 5
 *
 * LICENCE: This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 * @category  Plugin
 * @package   StatusNet
 * @author    Zach Copley <zach@status.net>
 * @author    Julien C <chaumond@gmail.com>
 * @author    Brion Vibber <brion@status.net>
 * @copyright 2009-2010 StatusNet, Inc.
 * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
 * @link      http://status.net/
 */

if (!defined('STATUSNET')) {
    exit(1);
}

require_once INSTALLDIR . '/plugins/TwitterBridge/twitter.php';

/**
 * Encapsulation of the Twitter status -> notice incoming bridge import.
 * Is used by both the polling twitterstatusfetcher.php daemon, and the
 * in-progress streaming import.
 *
 * @category Plugin
 * @package  StatusNet
 * @author   Zach Copley <zach@status.net>
 * @author   Julien C <chaumond@gmail.com>
 * @author   Brion Vibber <brion@status.net>
 * @license  http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
 * @link     http://status.net/
 * @link     http://twitter.com/
 */
class TwitterImport
{
    public function importStatus($status)
    {
        // Hacktastic: filter out stuff coming from this StatusNet
        $source = mb_strtolower(common_config('integration', 'source'));

        if (preg_match("/$source/", mb_strtolower($status->source))) {
            common_debug($this->name() . ' - Skipping import of status ' .
59
                         twitter_id($status) . ' with source ' . $source);
60
            return null;
61 62 63 64 65
        }

        // Don't save it if the user is protected
        // FIXME: save it but treat it as private
        if ($status->user->protected) {
66
            return null;
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
        }

        $notice = $this->saveStatus($status);

        return $notice;
    }

    function name()
    {
        return get_class($this);
    }

    function saveStatus($status)
    {
        $profile = $this->ensureProfile($status->user);

        if (empty($profile)) {
            common_log(LOG_ERR, $this->name() .
                ' - Problem saving notice. No associated Profile.');
            return null;
        }

89 90
        $statusId = twitter_id($status);
        $statusUri = $this->makeStatusURI($status->user->screen_name, $statusId);
91 92

        // check to see if we've already imported the status
93
        $n2s = Notice_to_status::getKV('status_id', $statusId);
94 95 96 97 98

        if (!empty($n2s)) {
            common_log(
                LOG_INFO,
                $this->name() .
99
                " - Ignoring duplicate import: {$statusId}"
100
            );
101
            return Notice::getKV('id', $n2s->notice_id);
102 103 104 105
        }

        // If it's a retweet, save it as a repeat!
        if (!empty($status->retweeted_status)) {
106
            common_log(LOG_INFO, "Status {$statusId} is a retweet of " . twitter_id($status->retweeted_status) . ".");
107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129
            $original = $this->saveStatus($status->retweeted_status);
            if (empty($original)) {
                return null;
            } else {
                $author = $original->getProfile();
                // TRANS: Message used to repeat a notice. RT is the abbreviation of 'retweet'.
                // TRANS: %1$s is the repeated user's name, %2$s is the repeated notice.
                $content = sprintf(_m('RT @%1$s %2$s'),
                                   $author->nickname,
                                   $original->content);

                if (Notice::contentTooLong($content)) {
                    $contentlimit = Notice::maxContent();
                    $content = mb_substr($content, 0, $contentlimit - 4) . ' ...';
                }

                $repeat = Notice::saveNew($profile->id,
                                          $content,
                                          'twitter',
                                          array('repeat_of' => $original->id,
                                                'uri' => $statusUri,
                                                'is_local' => Notice::GATEWAY));
                common_log(LOG_INFO, "Saved {$repeat->id} as a repeat of {$original->id}");
130
                Notice_to_status::saveNew($repeat->id, $statusId);
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
                return $repeat;
            }
        }

        $notice = new Notice();

        $notice->profile_id = $profile->id;
        $notice->uri        = $statusUri;
        $notice->url        = $statusUri;
        $notice->created    = strftime(
            '%Y-%m-%d %H:%M:%S',
            strtotime($status->created_at)
        );

        $notice->source     = 'twitter';

        $notice->reply_to   = null;

149 150 151
        $replyTo = twitter_id($status, 'in_reply_to_status_id');
        if (!empty($replyTo)) {
            common_log(LOG_INFO, "Status {$statusId} is a reply to status {$replyTo}");
152
            $n2s = Notice_to_status::getKV('status_id', $replyTo);
153
            if (empty($n2s)) {
154
                common_log(LOG_INFO, "Couldn't find local notice for status {$replyTo}");
155
            } else {
156
                $reply = Notice::getKV('id', $n2s->notice_id);
157
                if (empty($reply)) {
158
                    common_log(LOG_INFO, "Couldn't find local notice for status {$replyTo}");
159
                } else {
160
                    common_log(LOG_INFO, "Found local notice {$reply->id} for status {$replyTo}");
161 162 163 164 165 166 167 168 169
                    $notice->reply_to     = $reply->id;
                    $notice->conversation = $reply->conversation;
                }
            }
        }

        if (empty($notice->conversation)) {
            $conv = Conversation::create();
            $notice->conversation = $conv->id;
170
            common_log(LOG_INFO, "No known conversation for status {$statusId} so making a new one {$conv->id}.");
171 172 173 174
        }

        $notice->is_local   = Notice::GATEWAY;

175 176
        $notice->content  = html_entity_decode($this->linkify($status, FALSE), ENT_QUOTES, 'UTF-8');
        $notice->rendered = $this->linkify($status, TRUE);
177 178 179 180 181 182 183 184 185 186 187 188 189 190

        if (Event::handle('StartNoticeSave', array(&$notice))) {

            $id = $notice->insert();

            if (!$id) {
                common_log_db_error($notice, 'INSERT', __FILE__);
                common_log(LOG_ERR, $this->name() .
                    ' - Problem saving notice.');
            }

            Event::handle('EndNoticeSave', array($notice));
        }

191
        Notice_to_status::saveNew($notice->id, $statusId);
192 193

        $this->saveStatusMentions($notice, $status);
194
        $this->saveStatusAttachments($notice, $status);
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209

        $notice->blowOnInsert();

        return $notice;
    }

    /**
     * Make an URI for a status.
     *
     * @param object $status status object
     *
     * @return string URI
     */
    function makeStatusURI($username, $id)
    {
210
        return 'http://twitter.com/#!/'
211 212 213 214 215 216 217
          . $username
          . '/status/'
          . $id;
    }


    /**
218
     * Look up a Profile by profileurl field.  Profile::getKV() was
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266
     * not working consistently.
     *
     * @param string $nickname   local nickname of the Twitter user
     * @param string $profileurl the profile url
     *
     * @return mixed value the first Profile with that url, or null
     */
    function getProfileByUrl($nickname, $profileurl)
    {
        $profile = new Profile();
        $profile->nickname = $nickname;
        $profile->profileurl = $profileurl;
        $profile->limit(1);

        if ($profile->find()) {
            $profile->fetch();
            return $profile;
        }

        return null;
    }

    /**
     * Check to see if this Twitter status has already been imported
     *
     * @param Profile $profile   Twitter user's local profile
     * @param string  $statusUri URI of the status on Twitter
     *
     * @return mixed value a matching Notice or null
     */
    function checkDupe($profile, $statusUri)
    {
        $notice = new Notice();
        $notice->uri = $statusUri;
        $notice->profile_id = $profile->id;
        $notice->limit(1);

        if ($notice->find()) {
            $notice->fetch();
            return $notice;
        }

        return null;
    }

    function ensureProfile($user)
    {
        // check to see if there's already a profile for this user
267
        $profileurl = 'http://twitter.com/' . $user->screen_name;
268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307
        $profile = $this->getProfileByUrl($user->screen_name, $profileurl);

        if (!empty($profile)) {
            common_debug($this->name() .
                         " - Profile for $profile->nickname found.");

            // Check to see if the user's Avatar has changed

            $this->checkAvatar($user, $profile);
            return $profile;

        } else {
            common_debug($this->name() . ' - Adding profile and remote profile ' .
                         "for Twitter user: $profileurl.");

            $profile = new Profile();
            $profile->query("BEGIN");

            $profile->nickname = $user->screen_name;
            $profile->fullname = $user->name;
            $profile->homepage = $user->url;
            $profile->bio = $user->description;
            $profile->location = $user->location;
            $profile->profileurl = $profileurl;
            $profile->created = common_sql_now();

            try {
                $id = $profile->insert();
            } catch(Exception $e) {
                common_log(LOG_WARNING, $this->name() . ' Couldn\'t insert profile - ' . $e->getMessage());
            }

            if (empty($id)) {
                common_log_db_error($profile, 'INSERT', __FILE__);
                $profile->query("ROLLBACK");
                return false;
            }

            // check for remote profile

308
            $remote_pro = Remote_profile::getKV('uri', $profileurl);
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341

            if (empty($remote_pro)) {
                $remote_pro = new Remote_profile();

                $remote_pro->id = $id;
                $remote_pro->uri = $profileurl;
                $remote_pro->created = common_sql_now();

                try {
                    $rid = $remote_pro->insert();
                } catch (Exception $e) {
                    common_log(LOG_WARNING, $this->name() . ' Couldn\'t save remote profile - ' . $e->getMessage());
                }

                if (empty($rid)) {
                    common_log_db_error($profile, 'INSERT', __FILE__);
                    $profile->query("ROLLBACK");
                    return false;
                }
            }

            $profile->query("COMMIT");

            $this->saveAvatars($user, $id);

            return $profile;
        }
    }

    function checkAvatar($twitter_user, $profile)
    {
        global $config;

342
        $newname = 'Twitter_' . $twitter_user->id . '_' . basename($twitter_user->profile_image_url);
343

344 345 346 347 348 349 350 351
        try {
            $avatar = $profile->getAvatar(AVATAR_STREAM_SIZE);
            $oldname = $avatar->filename;
            unset($avatar);
        } catch (Exception $e) {
            $oldname = null;
        }
        
352 353 354 355 356 357 358 359 360

        if ($newname != $oldname) {
            common_debug($this->name() . ' - Avatar for Twitter user ' .
                         "$profile->nickname has changed.");
            common_debug($this->name() . " - old: $oldname new: $newname");

            $this->updateAvatars($twitter_user, $profile);
        }

361
        if (Avatar::hasUploaded($profile)) {
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376
            common_debug($this->name() . ' - Twitter user ' .
                         $profile->nickname .
                         ' is missing one or more local avatars.');
            common_debug($this->name() ." - old: $oldname new: $newname");

            $this->updateAvatars($twitter_user, $profile);
        }
    }

    function updateAvatars($twitter_user, $profile) {

        global $config;

        $path_parts = pathinfo($twitter_user->profile_image_url);

377 378
        $ext = (isset($path_parts['extension']) ? '.'.$path_parts['extension'] : '');	// some lack extension
        $img_root = basename($path_parts['basename'], '_normal'.$ext);	// cut off extension
379
        $mediatype = $this->getMediatype(substr($ext, 1));
380 381 382

        foreach (array('mini', 'normal', 'bigger') as $size) {
            $url = $path_parts['dirname'] . '/' .
383
                $img_root . '_' . $size . $ext;
384
            $filename = 'Twitter_' . $twitter_user->id . '_' .
385
                $img_root . '_' . $size . $ext;
386 387 388 389 390 391 392 393 394 395 396

            $this->updateAvatar($profile->id, $size, $mediatype, $filename);
            $this->fetchAvatar($url, $filename);
        }
    }

    function getMediatype($ext)
    {
        $mediatype = null;

        switch (strtolower($ext)) {
397
        case 'jpeg':
398
        case 'jpg':
mmn's avatar
mmn committed
399
            $mediatype = 'image/jpeg';
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
            break;
        case 'gif':
            $mediatype = 'image/gif';
            break;
        default:
            $mediatype = 'image/png';
        }

        return $mediatype;
    }

    function saveAvatars($user, $id)
    {
        global $config;

        $path_parts = pathinfo($user->profile_image_url);
416 417
        $ext = (isset($path_parts['extension']) ? '.'.$path_parts['extension'] : '');
        $img_root = basename($path_parts['basename'], '_normal'.$ext);
418
        $mediatype = $this->getMediatype(substr($ext, 1));
419 420 421

        foreach (array('mini', 'normal', 'bigger') as $size) {
            $url = $path_parts['dirname'] . '/' .
422
                $img_root . '_' . $size . $ext;
423
            $filename = 'Twitter_' . $user->id . '_' .
424
                $img_root . '_' . $size . $ext;
425 426 427 428 429 430 431 432 433 434 435 436 437 438

            if ($this->fetchAvatar($url, $filename)) {
                $this->newAvatar($id, $size, $mediatype, $filename);
            } else {
                common_log(LOG_WARNING, $id() .
                           " - Problem fetching Avatar: $url");
            }
        }
    }

    function updateAvatar($profile_id, $size, $mediatype, $filename) {

        common_debug($this->name() . " - Updating avatar: $size");

439
        $profile = Profile::getKV($profile_id);
440 441 442 443 444 445

        if (empty($profile)) {
            common_debug($this->name() . " - Couldn't get profile: $profile_id!");
            return;
        }

446
        Avatar::deleteFromProfile($profile);
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529

        $this->newAvatar($profile->id, $size, $mediatype, $filename);
    }

    function newAvatar($profile_id, $size, $mediatype, $filename)
    {
        global $config;

        $avatar = new Avatar();
        $avatar->profile_id = $profile_id;

        switch($size) {
        case 'mini':
            $avatar->width  = 24;
            $avatar->height = 24;
            break;
        case 'normal':
            $avatar->width  = 48;
            $avatar->height = 48;
            break;
        default:
            // Note: Twitter's big avatars are a different size than
            // StatusNet's (StatusNet's = 96)
            $avatar->width  = 73;
            $avatar->height = 73;
        }

        $avatar->original = 0; // we don't have the original
        $avatar->mediatype = $mediatype;
        $avatar->filename = $filename;
        $avatar->url = Avatar::url($filename);

        $avatar->created = common_sql_now();

        try {
            $id = $avatar->insert();
        } catch (Exception $e) {
            common_log(LOG_WARNING, $this->name() . ' Couldn\'t insert avatar - ' . $e->getMessage());
        }

        if (empty($id)) {
            common_log_db_error($avatar, 'INSERT', __FILE__);
            return null;
        }

        common_debug($this->name() .
                     " - Saved new $size avatar for $profile_id.");

        return $id;
    }

    /**
     * Fetch a remote avatar image and save to local storage.
     *
     * @param string $url avatar source URL
     * @param string $filename bare local filename for download
     * @return bool true on success, false on failure
     */
    function fetchAvatar($url, $filename)
    {
        common_debug($this->name() . " - Fetching Twitter avatar: $url");

        $request = HTTPClient::start();
        $response = $request->get($url);
        if ($response->isOk()) {
            $avatarfile = Avatar::path($filename);
            $ok = file_put_contents($avatarfile, $response->getBody());
            if (!$ok) {
                common_log(LOG_WARNING, $this->name() .
                           " - Couldn't open file $filename");
                return false;
            }
        } else {
            return false;
        }

        return true;
    }

    const URL = 1;
    const HASHTAG = 2;
    const MENTION = 3;

530
    function linkify($status, $html = FALSE)
531 532 533 534
    {
        $text = $status->text;

        if (empty($status->entities)) {
535 536
            $statusId = twitter_id($status);
            common_log(LOG_WARNING, "No entities data for {$statusId}; trying to fake up links ourselves.");
537 538 539 540 541 542 543
            $text = common_replace_urls_callback($text, 'common_linkify');
            $text = preg_replace('/(^|\&quot\;|\'|\(|\[|\{|\s+)#([\pL\pN_\-\.]{1,64})/e', "'\\1#'.TwitterStatusFetcher::tagLink('\\2')", $text);
            $text = preg_replace('/(^|\s+)@([a-z0-9A-Z_]{1,64})/e', "'\\1@'.TwitterStatusFetcher::atLink('\\2')", $text);
            return $text;
        }

        // Move all the entities into order so we can
544 545
        // replace them and escape surrounding plaintext
        // in order
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566

        $toReplace = array();

        if (!empty($status->entities->urls)) {
            foreach ($status->entities->urls as $url) {
                $toReplace[$url->indices[0]] = array(self::URL, $url);
            }
        }

        if (!empty($status->entities->hashtags)) {
            foreach ($status->entities->hashtags as $hashtag) {
                $toReplace[$hashtag->indices[0]] = array(self::HASHTAG, $hashtag);
            }
        }

        if (!empty($status->entities->user_mentions)) {
            foreach ($status->entities->user_mentions as $mention) {
                $toReplace[$mention->indices[0]] = array(self::MENTION, $mention);
            }
        }

567
        // sort in forward order by key
568

569 570 571 572
        ksort($toReplace);

        $result = '';
        $cursor = 0;
573 574 575

        foreach ($toReplace as $part) {
            list($type, $object) = $part;
576 577 578 579 580 581 582 583
            $start = $object->indices[0];
            $end = $object->indices[1];
            if ($cursor < $start) {
                // Copy in the preceding plaintext
                $result .= $this->twitEscape(mb_substr($text, $cursor, $start - $cursor));
                $cursor = $start;
            }
            $orig = $this->twitEscape(mb_substr($text, $start, $end - $start));
584 585
            switch($type) {
            case self::URL:
586
                $linkText = $this->makeUrlLink($object, $orig, $html);
587 588
                break;
            case self::HASHTAG:
589 590 591 592 593
                if ($html) {
                    $linkText = $this->makeHashtagLink($object, $orig);
                }else{
                    $linkText = $orig;
                }
594 595
                break;
            case self::MENTION:
596 597 598 599 600
                if ($html) {
                    $linkText = $this->makeMentionLink($object, $orig);
                }else{
                    $linkText = $orig;
                }
601 602
                break;
            default:
603
                $linkText = $orig;
604 605
                continue;
            }
606 607
            $result .= $linkText;
            $cursor = $end;
608
        }
609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
        $last = $this->twitEscape(mb_substr($text, $cursor));
        $result .= $last;

        return $result;
    }

    function twitEscape($str)
    {
        // Twitter seems to preemptive turn < and > into &lt; and &gt;
        // but doesn't for &, so while you may have some magic protection
        // against XSS by not bothing to escape manually, you still get
        // invalid XHTML. Thanks!
        //
        // Looks like their web interface pretty much sends anything
        // through intact, so.... to do equivalent, decode all entities
        // and then re-encode the special ones.
        return htmlspecialchars(html_entity_decode($str, ENT_COMPAT, 'UTF-8'));
626 627
    }

628
    function makeUrlLink($object, $orig, $html)
629
    {
630 631 632 633 634
        if ($html) {
            return '<a href="'.htmlspecialchars($object->expanded_url).'" class="extlink">'.htmlspecialchars($object->display_url).'</a>';
        }else{
            return htmlspecialchars($object->expanded_url);
        }
635 636
    }

637
    function makeHashtagLink($object, $orig)
638
    {
639
        return "#" . self::tagLink($object->text, substr($orig, 1));
640 641
    }

642
    function makeMentionLink($object, $orig)
643
    {
644
        return "@".self::atLink($object->screen_name, $object->name, substr($orig, 1));
645 646
    }

647
    static function tagLink($tag, $orig)
648
    {
649
        return "<a href='https://search.twitter.com/search?q=%23{$tag}' class='hashtag'>{$orig}</a>";
650 651
    }

652
    static function atLink($screenName, $fullName, $orig)
653 654
    {
        if (!empty($fullName)) {
655
            return "<a href='http://twitter.com/#!/{$screenName}' title='{$fullName}'>{$orig}</a>";
656
        } else {
657
            return "<a href='http://twitter.com/#!/{$screenName}'>{$orig}</a>";
658 659 660 661 662 663 664 665 666 667 668 669 670 671
        }
    }

    function saveStatusMentions($notice, $status)
    {
        $mentions = array();

        if (empty($status->entities) || empty($status->entities->user_mentions)) {
            return;
        }

        foreach ($status->entities->user_mentions as $mention) {
            $flink = Foreign_link::getByForeignID($mention->id, TWITTER_SERVICE);
            if (!empty($flink)) {
672
                $user = User::getKV('id', $flink->user_id);
673 674 675 676
                if (!empty($user)) {
                    $reply = new Reply();
                    $reply->notice_id  = $notice->id;
                    $reply->profile_id = $user->id;
677
                    $reply->modified   = $notice->created;
678 679 680 681 682 683
                    common_log(LOG_INFO, __METHOD__ . ": saving reply: notice {$notice->id} to profile {$user->id}");
                    $id = $reply->insert();
                }
            }
        }
    }
684 685 686 687 688 689 690 691 692 693

    /**
     * Record URL links from the notice. Needed to get thumbnail records
     * for referenced photo and video posts, etc.
     *
     * @param Notice $notice
     * @param object $status
     */
    function saveStatusAttachments($notice, $status)
    {
694 695 696 697 698
        if (common_config('attachments', 'process_links')) {
            if (!empty($status->entities) && !empty($status->entities->urls)) {
                foreach ($status->entities->urls as $url) {
                    File::processNew($url->url, $notice->id);
                }
699 700 701
            }
        }
    }
702
}