Skip to content

GitLab

G
gnu-social
Commit 8f438da2 authored by Brion Vibber's avatar Brion Vibber
Browse files
Options

Ok, command-line workflow for YammerImportPlugin seems to mostly work, at... 

Ok, command-line workflow for YammerImportPlugin seems to mostly work, at least on tiny test site :D
parent ae507b04
Hide whitespace changes
Inline Side-by-side
Showing with 144 additions and 63 deletions
plugins/YammerImport/classes/Yammer_common.php
View file @ 8f438da2
......@@ -138,13 +138,13 @@ class Yammer_common extends Memcached_DataObject
protected static function doRecord($class, $field, $orig_id, $local_id)
{
$map = self::staticGet('id', $orig_id);
$map = parent::staticGet($class, 'id', $orig_id);
if (!empty($map)) {
return $map;
}
$map = self::staticGet($field, $local_id);
$map = parent::staticGet($class, $field, $local_id);
if (!empty($map)) {
return $map;
......
plugins/YammerImport/classes/Yammer_notice_stub.php
View file @ 8f438da2
......@@ -48,6 +48,23 @@ class Yammer_notice_stub extends Memcached_DataObject
public $json_data; // text
public $created; // datetime
/**
* Get an instance by key
*
* This is a utility method to get a single instance with a given key value.
*
* @param string $k Key to use to lookup
* @param mixed $v Value to lookup
*
* @return Yammer_notice_stub object found, or null for no hits
*
*/
function staticGet($k, $v=null)
{
return Memcached_DataObject::staticGet('Yammer_notice_stub', $k, $v);
}
/**
* Return schema definition to set this table up in onCheckSchema
*/
......@@ -126,6 +143,16 @@ class Yammer_notice_stub extends Memcached_DataObject
return array(false, false, false);
}
/**
* Decode the stored data structure.
*
* @return mixed
*/
public function getData()
{
return json_decode($this->json_data, true);
}
/**
* Save the native Yammer API representation of a message for the pending
* import. Since they come in in reverse chronological order, we need to
......@@ -153,22 +180,4 @@ class Yammer_notice_stub extends Memcached_DataObject
return $stub;
}
/**
* Save a mapping between a remote Yammer and local imported user.
*
* @param integer $user_id ID of the status in StatusNet
*
* @return Yammer_notice_stub new object for this value
*/
static function retrieve($orig_id)
{
$stub = self::staticGet('id', $orig_id);
if ($stub) {
return json_decode($stub->json_data, true);
} else {
return false;
}
}
}
plugins/YammerImport/classes/Yammer_state.php
View file @ 8f438da2
......@@ -36,7 +36,6 @@ class Yammer_state extends Memcached_DataObject
public $__table = 'yammer_state'; // table name
public $id; // int primary_key not_null
public $state; // import state key
public $request_token; // oauth request token; clear when auth is complete.
public $oauth_token; // actual oauth token! clear when import is done?
public $oauth_secret; // actual oauth secret! clear when import is done?
public $users_page; // last page of users we've fetched
......@@ -46,6 +45,23 @@ class Yammer_state extends Memcached_DataObject
public $created; // datetime
public $modified; // datetime
/**
* Get an instance by key
*
* This is a utility method to get a single instance with a given key value.
*
* @param string $k Key to use to lookup
* @param mixed $v Value to lookup
*
* @return Yammer_state object found, or null for no hits
*
*/
function staticGet($k, $v=null)
{
return Memcached_DataObject::staticGet('Yammer_state', $k, $v);
}
/**
* Return schema definition to set this table up in onCheckSchema
*/
......@@ -54,7 +70,6 @@ class Yammer_state extends Memcached_DataObject
return array(new ColumnDef('id', 'int', null,
false, 'PRI'),
new ColumnDef('state', 'text'),
new ColumnDef('request_token', 'text'),
new ColumnDef('oauth_token', 'text'),
new ColumnDef('oauth_secret', 'text'),
new ColumnDef('users_page', 'int'),
......@@ -78,7 +93,6 @@ class Yammer_state extends Memcached_DataObject
{
return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL,
'state' => DB_DATAOBJECT_STR,
'request_token' => DB_DATAOBJECT_STR,
'oauth_token' => DB_DATAOBJECT_STR,
'oauth_secret' => DB_DATAOBJECT_STR,
'users_page' => DB_DATAOBJECT_INT,
......
plugins/YammerImport/lib/sn_yammerclient.php
View file @ 8f438da2
......@@ -104,7 +104,8 @@ class SN_YammerClient
{
$body = $this->fetchApi("api/v1/$method.json", $params);
$data = json_decode($body, true);
if (!$data) {
if ($data === null) {
common_log(LOG_ERR, "Invalid JSON response from Yammer API: " . $body);
throw new Exception("Invalid JSON response from Yammer API");
}
return $data;
......@@ -161,7 +162,7 @@ class SN_YammerClient
if ($this->token || $this->tokenSecret) {
throw new Exception("Requesting a token, but already set up with a token");
}
$data = $this->fetch('oauth/request_token');
$data = $this->fetchApi('oauth/request_token');
$arr = array();
parse_str($data, $arr);
return $arr;
......@@ -176,7 +177,7 @@ class SN_YammerClient
public function accessToken($verifier)
{
$this->verifier = $verifier;
$data = $this->fetch('oauth/access_token');
$data = $this->fetchApi('oauth/access_token');
$this->verifier = null;
$arr = array();
parse_str($data, $arr);
......
plugins/YammerImport/lib/yammerimporter.php
View file @ 8f438da2
......@@ -327,17 +327,20 @@ class YammerImporter
private function findImportedUser($origId)
{
return Yammer_user::staticGet('id', $origId);
$map = Yammer_user::staticGet('id', $origId);
return $map ? $map->user_id : null;
}
private function findImportedGroup($origId)
{
return Yammer_group::staticGet('id', $origId);
$map = Yammer_group::staticGet('id', $origId);
return $map ? $map->group_id : null;
}
private function findImportedNotice($origId)
{
return Yammer_notice::staticGet('id', $origId);
$map = Yammer_notice::staticGet('id', $origId);
return $map ? $map->notice_id : null;
}
private function recordImportedUser($origId, $userId)
......@@ -370,7 +373,7 @@ class YammerImporter
// Blaaaaaarf!
$known = array('Pacific Time (US & Canada)' => 'America/Los_Angeles',
'Eastern Time (US & Canada)' => 'America/New_York');
if (array_key_exists($known, $tz)) {
if (array_key_exists($tz, $known)) {
return $known[$tz];
} else {
return false;
......
plugins/YammerImport/lib/yammerrunner.php
View file @ 8f438da2
......@@ -33,18 +33,31 @@ class YammerRunner
private $client;
private $importer;
/**
* Normalize our singleton state and give us a YammerRunner object to play with!
*
* @return YammerRunner
*/
public static function init()
{
$state = Yammer_state::staticGet('id', 1);
if (!$state) {
$state = new Yammer_state();
$state->id = 1;
$state->state = 'init';
$state->insert();
$state = self::initState();
}
return new YammerRunner($state);
}
private static function initState()
{
$state = new Yammer_state();
$state->id = 1;
$state->state = 'init';
$state->created = common_sql_now();
$state->modified = common_sql_now();
$state->insert();
return $state;
}
private function __construct($state)
{
$this->state = $state;
......@@ -55,7 +68,7 @@ class YammerRunner
$this->state->oauth_token,
$this->state->oauth_secret);
$this->importer = new YammerImporter($client);
$this->importer = new YammerImporter($this->client);
}
/**
......@@ -81,6 +94,8 @@ class YammerRunner
/**
* Check if we have work to do in iterate().
*
* @return boolean
*/
public function hasWork()
{
......@@ -88,6 +103,15 @@ class YammerRunner
return in_array($this->state(), $workStates);
}
/**
* Blow away any current state!
*/
public function reset()
{
$this->state->delete();
$this->state = self::initState();
}
/**
* Start the authentication process! If all goes well, we'll get back a URL.
* Have the user visit that URL, log in on Yammer and verify the importer's
......@@ -102,12 +126,16 @@ class YammerRunner
throw ServerError("Cannot request Yammer auth; already there!");
}
$data = $this->client->requestToken();
$old = clone($this->state);
$this->state->state = 'requesting-auth';
$this->state->request_token = $client->requestToken();
$this->state->oauth_token = $data['oauth_token'];
$this->state->oauth_secret = $data['oauth_token_secret'];
$this->state->modified = common_sql_now();
$this->state->update($old);
return $this->client->authorizeUrl($this->state->request_token);
return $this->client->authorizeUrl($this->state->oauth_token);
}
/**
......@@ -127,12 +155,13 @@ class YammerRunner
throw ServerError("Cannot save auth token in Yammer import state {$this->state->state}");
}
$old = clone($this->state);
list($token, $secret) = $this->client->getAuthToken($verifier);
$this->state->verifier = '';
$this->state->oauth_token = $token;
$this->state->oauth_secret = $secret;
$data = $this->client->accessToken($verifier);
$old = clone($this->state);
$this->state->state = 'import-users';
$this->state->oauth_token = $data['oauth_token'];
$this->state->oauth_secret = $data['oauth_token_secret'];
$this->state->modified = common_sql_now();
$this->state->update($old);
return true;
......@@ -146,8 +175,7 @@ class YammerRunner
*/
public function iterate()
{
switch($state->state)
switch($this->state())
{
case 'init':
case 'requesting-auth':
......@@ -188,11 +216,12 @@ class YammerRunner
$this->state->state = 'import-groups';
} else {
foreach ($data as $item) {
$user = $imp->importUser($item);
$user = $this->importer->importUser($item);
common_log(LOG_INFO, "Imported Yammer user " . $item['id'] . " as $user->nickname ($user->id)");
}
$this->state->users_page = $page;
}
$this->state->modified = common_sql_now();
$this->state->update($old);
return true;
}
......@@ -214,14 +243,15 @@ class YammerRunner
if (count($data) == 0) {
common_log(LOG_INFO, "Finished importing Yammer groups; moving on to messages.");
$this->state->state = 'import-messages';
$this->state->state = 'fetch-messages';
} else {
foreach ($data as $item) {
$group = $imp->importGroup($item);
$group = $this->importer->importGroup($item);
common_log(LOG_INFO, "Imported Yammer group " . $item['id'] . " as $group->nickname ($group->id)");
}
$this->state->groups_page = $page;
}
$this->state->modified = common_sql_now();
$this->state->update($old);
return true;
}
......@@ -248,16 +278,17 @@ class YammerRunner
$data = $this->client->messages($params);
$messages = $data['messages'];
if (count($data) == 0) {
if (count($messages) == 0) {
common_log(LOG_INFO, "Finished fetching Yammer messages; moving on to save messages.");
$this->state->state = 'save-messages';
} else {
foreach ($data as $item) {
foreach ($messages as $item) {
Yammer_notice_stub::record($item['id'], $item);
$oldest = $item['id'];
}
$this->state->messages_oldest = $oldest;
}
$this->state->modified = common_sql_now();
$this->state->update($old);
return true;
}
......@@ -267,10 +298,13 @@ class YammerRunner
$old = clone($this->state);
$newest = intval($this->state->messages_newest);
$stub = new Yammer_notice_stub();
if ($newest) {
$stub->addWhere('id > ' . $newest);
$stub->whereAdd('id > ' . $newest);
}
$stub->limit(20);
$stub->orderBy('id');
$stub->find();
if ($stub->N == 0) {
......@@ -278,13 +312,14 @@ class YammerRunner
$this->state->state = 'done';
} else {
while ($stub->fetch()) {
$item = json_decode($stub->json_data);
$item = $stub->getData();
$notice = $this->importer->importNotice($item);
common_log(LOG_INFO, "Imported Yammer notice " . $item['id'] . " as $notice->id");
$newest = $item['id'];
}
$this->state->messages_newest = $newest;
}
$this->state->modified = common_sql_now();
$this->state->update($old);
return true;
}
......
plugins/YammerImport/scripts/yammer-import.php
View file @ 8f438da2
......@@ -4,41 +4,60 @@ if (php_sapi_name() != 'cli') {
die('no');
}
define('INSTALLDIR', dirname(dirname(dirname(dirname(__FILE__)))));
$longoptions = array('verify=', 'reset');
require INSTALLDIR . "/scripts/commandline.inc";
echo "Checking current state...\n";
$runner = YammerRunner::init();
if (have_option('reset')) {
echo "Resetting Yammer import state...\n";
$runner->reset();
}
switch ($runner->state())
{
case 'init':
echo "Requesting authentication to Yammer API...\n";
$url = $runner->requestAuth();
echo "Log in to Yammer at the following URL and confirm permissions:\n";
echo "\n";
echo " $url\n";
echo "\n";
echo "Pass the resulting code back by running:\n"
echo "\n"
echo " php yammer-import.php --auth=####\n";
echo "Pass the resulting code back by running:\n";
echo "\n";
echo " php yammer-import.php --verify=####\n";
echo "\n";
break;
case 'requesting-auth':
if (empty($options['auth'])) {
echo "Please finish authenticating!\n";
break;
if (!have_option('verify')) {
echo "Awaiting authentication...\n";
echo "\n";
echo "If you need to start over, reset the state:\n";
echo "\n";
echo " php yammer-import.php --reset\n";
echo "\n";
exit(1);
}
$runner->saveAuthToken($options['auth']);
echo "Saving final authentication token for Yammer API...\n";
$runner->saveAuthToken(get_option_value('verify'));
// Fall through...
default:
while (true) {
echo "... {$runner->state->state}\n";
while ($runner->hasWork()) {
echo "... {$runner->state()}\n";
if (!$runner->iterate()) {
echo "... done.\n";
break;
echo "FAIL??!?!?!\n";
}
}
if ($runner->isDone()) {
echo "... done.\n";
} else {
echo "... no more import work scheduled.\n";
}
break;
}
\ No newline at end of file
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

git.gnu.io is graciously hosted by the Free Software Foundation. https://donate.fsf.org