Work in progress: most of the infrastructure for running import via BG queues...

Work in progress: most of the infrastructure for running import via BG queues or CLI script is now in place (untested, no UI, needs tweaks & fixes)

plugins/YammerImport/README

@@ -73,3 +73,45 @@ they do on Yammer; they will be linked instead.
 File type and size limitations on attachments will be applied, so beware some
 attachments may not make it through.
 
+
+
+
+Code structure
+==============
+
+Standalone classes
+------------------
+
+YammerRunner: encapsulates the iterative process of retrieving the various users,
+              groups, and messages via SN_YammerClient and saving them locally
+              via YammerImporter.
+
+SN_YammerClient: encapsulates HTTP+OAuth interface to Yammer API, returns data
+                 as straight decoded JSON object trees.
+
+YammerImporter: encapsulates logic to pull information from the returned API data
+                and convert them to native StatusNet users, groups, and messages.
+
+Web UI actions
+-------------
+
+YammeradminpanelAction: web panel for site administrator to initiate and monitor
+                        the import process.
+
+Command-line scripts
+--------------------
+
+yammer-import.php: CLI script to start a Yammer import run in one go.
+
+Database objects
+----------------
+
+Yammer_state: data object storing YammerRunner's state between iterations.
+
+Yammer_notice_stub: data object for temporary storage of fetched Yammer messages
+                    between fetching them (reverse chron order) and saving them
+                    to local messages (forward chron order).
+Yammer_user,
+Yammer_group,
+Yammer_notice: data objects mapping original Yammer item IDs to their local copies.
+

plugins/YammerImport/lib/yammerqueuehandler.php

@@ -36,11 +36,23 @@ class YammerQueueHandler extends QueueHandler
 
     function handle($notice)
     {
-        $importer = new YammerImporter();
-        if ($importer->hasWork()) {
-            return $importer->iterate();
+        $runner = YammerRunner::init();
+        if ($runner->hasWork()) {
+            if ($runner->iterate()) {
+                if ($runner->hasWork()) {
+                    // More to do? Shove us back on the queue...
+                    $qm = QueueManager::get();
+                    $qm->enqueue('YammerImport', 'yammer');
+                }
+                return true;
+            } else {
+                // Something failed?
+                // @fixme should we be trying again here, or should we give warning?
+                return false;
+            }
         } else {
             // We're done!
+            common_log(LOG_INFO, "Yammer import has no work to do at this time; discarding.");
             return true;
         }
     }

plugins/YammerImport/lib/yammerrunner.php

@@ -33,29 +33,123 @@ class YammerRunner
     private $client;
     private $importer;
 
-    function __construct()
+    public static function init()
     {
         $state = Yammer_state::staticGet('id', 1);
         if (!$state) {
-            common_log(LOG_ERR, "No YammerImport state during import run. Should not happen!");
-            throw new ServerException('No YammerImport state during import run.');
+            $state = new Yammer_state();
+            $state->id = 1;
+            $state->state = 'init';
+            $state->insert();
         }
+        return new YammerRunner($state);
+    }
 
+    private function __construct($state)
+    {
         $this->state = $state;
+
         $this->client = new SN_YammerClient(
             common_config('yammer', 'consumer_key'),
             common_config('yammer', 'consumer_secret'),
             $this->state->oauth_token,
             $this->state->oauth_secret);
+
         $this->importer = new YammerImporter($client);
     }
 
+    /**
+     * Check which state we're in
+     *
+     * @return string
+     */
+    public function state()
+    {
+        return $this->state->state;
+    }
+
+    /**
+     * Is the import done, finished, complete, finito?
+     *
+     * @return boolean
+     */
+    public function isDone()
+    {
+        $workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
+        return ($this->state() == 'done');
+    }
+
+    /**
+     * Check if we have work to do in iterate().
+     */
+    public function hasWork()
+    {
+        $workStates = array('import-users', 'import-groups', 'fetch-messages', 'save-messages');
+        return in_array($this->state(), $workStates);
+    }
+
+    /**
+     * Start the authentication process! If all goes well, we'll get back a URL.
+     * Have the user visit that URL, log in on Yammer and verify the importer's
+     * permissions. They'll get back a verification code, which needs to be passed
+     * on to saveAuthToken().
+     *
+     * @return string URL
+     */
+    public function requestAuth()
+    {
+        if ($this->state->state != 'init') {
+            throw ServerError("Cannot request Yammer auth; already there!");
+        }
+
+        $old = clone($this->state);
+        $this->state->state = 'requesting-auth';
+        $this->state->request_token = $client->requestToken();
+        $this->state->update($old);
+
+        return $this->client->authorizeUrl($this->state->request_token);
+    }
+
+    /**
+     * Now that the user's given us this verification code from Yammer, we can
+     * request a final OAuth token/secret pair which we can use to access the
+     * API.
+     *
+     * After success here, we'll be ready to move on and run through iterate()
+     * until the import is complete.
+     *
+     * @param string $verifier
+     * @return boolean success
+     */
+    public function saveAuthToken($verifier)
+    {
+        if ($this->state->state != 'requesting-auth') {
+            throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
+        }
+
+        $old = clone($this->state);
+        list($token, $secret) = $this->client->getAuthToken($verifier);
+        $this->state->verifier = '';
+        $this->state->oauth_token = $token;
+        $this->state->oauth_secret = $secret;
+
+        $this->state->update($old);
+
+        return true;
+    }
+
+    /**
+     * Once authentication is complete, we need to call iterate() a bunch of times
+     * until state() returns 'done'.
+     *
+     * @return boolean success
+     */
     public function iterate()
     {
 
         switch($state->state)
         {
-            case null:
+            case 'init':
             case 'requesting-auth':
                 // Neither of these should reach our background state!
                 common_log(LOG_ERR, "Non-background YammerImport state '$state->state' during import run!");

plugins/YammerImport/scripts/yammer-import.php

@@ -8,34 +8,37 @@ define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
 
 require INSTALLDIR . "/scripts/commandline.inc";
 
-// temp stuff
-require 'yam-config.php';
-$yam = new SN_YammerClient($consumerKey, $consumerSecret, $token, $tokenSecret);
-$imp = new YammerImporter($yam);
+$runner = YammerRunner::init();
 
-// First, import all the users!
-// @fixme follow paging -- we only get 50 at a time
-$data = $yam->users();
-foreach ($data as $item) {
-    $user = $imp->importUser($item);
-    echo "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)\n";
-}
+switch ($runner->state())
+{
+    case 'init':
+        $url = $runner->requestAuth();
+        echo "Log in to Yammer at the following URL and confirm permissions:\n";
+        echo "\n";
+        echo "    $url\n";
+        echo "\n";
+        echo "Pass the resulting code back by running:\n"
+        echo "\n"
+        echo "    php yammer-import.php --auth=####\n";
+        echo "\n";
+        break;
 
-// Groups!
-// @fixme follow paging -- we only get 20 at a time
-$data = $yam->groups();
-foreach ($data as $item) {
-    $group = $imp->importGroup($item);
-    echo "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)\n";
-}
+    case 'requesting-auth':
+        if (empty($options['auth'])) {
+            echo "Please finish authenticating!\n";
+            break;
+        }
+        $runner->saveAuthToken($options['auth']);
+        // Fall through...
 
-// Messages!
-// Process in reverse chron order...
-// @fixme follow paging -- we only get 20 at a time, and start at the most recent!
-$data = $yam->messages();
-$messages = $data['messages'];
-$messages = array_reverse($messages);
-foreach ($messages as $item) {
-    $notice = $imp->importNotice($item);
-    echo "Imported Yammer notice " . $item['id'] . " as $notice->id\n";
-}
+    default:
+        while (true) {
+            echo "... {$runner->state->state}\n";
+            if (!$runner->iterate()) {
+                echo "... done.\n";
+                break;
+            }
+        }
+        break;
+}
\ No newline at end of file