Get sync-lists command to create missing entries on MAL

This commit is contained in:
Timothy Warren 2017-02-14 15:29:13 -05:00
parent 9c1dc50e65
commit 0d553b7dd4
9 changed files with 275 additions and 131 deletions

View File

@ -20,7 +20,7 @@ use const Aviat\AnimeClient\SESSION_SEGMENT;
use function Amp\wait; use function Amp\wait;
use Amp\Artax\Client; use Amp\Artax\{Client, Request};
use Aviat\AnimeClient\AnimeClient; use Aviat\AnimeClient\AnimeClient;
use Aviat\AnimeClient\API\Kitsu as K; use Aviat\AnimeClient\API\Kitsu as K;
use Aviat\Ion\Json; use Aviat\Ion\Json;
@ -53,9 +53,9 @@ trait KitsuTrait {
* @param string $type * @param string $type
* @param string $url * @param string $url
* @param array $options * @param array $options
* @return \Amp\Artax\Response * @return \Amp\Artax\Request
*/ */
public function setUpRequest(string $type, string $url, array $options = []) public function setUpRequest(string $type, string $url, array $options = []): Request
{ {
$config = $this->container->get('config'); $config = $this->container->get('config');
@ -69,7 +69,6 @@ trait KitsuTrait {
{ {
$token = $sessionSegment->get('auth_token'); $token = $sessionSegment->get('auth_token');
$request = $request->setAuth('bearer', $token); $request = $request->setAuth('bearer', $token);
// $defaultOptions['headers']['Authorization'] = "bearer {$token}";
} }
if (array_key_exists('form_params', $options)) if (array_key_exists('form_params', $options))
@ -138,7 +137,7 @@ trait KitsuTrait {
{ {
if ($logger) if ($logger)
{ {
$logger->warning('Non 200 response for api call', $response->getBody()); $logger->warning('Non 200 response for api call', (array)$response->getBody());
} }
} }

View File

@ -203,15 +203,56 @@ class Model {
$baseData = $this->getRawMediaData('manga', $mangaId); $baseData = $this->getRawMediaData('manga', $mangaId);
return $this->mangaTransformer->transform($baseData); return $this->mangaTransformer->transform($baseData);
} }
/**
* Get the number of anime list items
*
* @return int
*/
public function getAnimeListCount() : int
{
$options = [
'query' => [
'filter' => [
'user_id' => $this->getUserIdByUsername(),
'media_type' => 'Anime'
],
'page' => [
'limit' => 1
],
'sort' => '-updated_at'
]
];
$response = $this->getRequest('library-entries', $options);
return $response['meta']['count'];
}
/** /**
* Get and transform the entirety of the user's anime list * Get and transform the entirety of the user's anime list
* *
* @return array * @return Request
*/ */
public function getFullAnimeList(): array public function getFullAnimeList(int $limit = 100, int $offset = 0): Request
{ {
$options = [
'query' => [
'filter' => [
'user_id' => $this->getUserIdByUsername($this->getUsername()),
'media_type' => 'Anime'
],
'include' => 'anime.mappings',
'page' => [
'offset' => $offset,
'limit' => $limit
],
'sort' => '-updated_at'
]
];
return $this->setUpRequest('GET', 'library-entries', $options);
} }
/** /**

View File

@ -36,6 +36,14 @@ class MAL {
KAWS::DROPPED => AnimeWatchingStatus::DROPPED, KAWS::DROPPED => AnimeWatchingStatus::DROPPED,
KAWS::PLAN_TO_WATCH => AnimeWatchingStatus::PLAN_TO_WATCH KAWS::PLAN_TO_WATCH => AnimeWatchingStatus::PLAN_TO_WATCH
]; ];
const MAL_KITSU_WATCHING_STATUS_MAP = [
1 => KAWS::WATCHING,
2 => KAWS::COMPLETED,
3 => KAWS::ON_HOLD,
4 => KAWS::DROPPED,
6 => KAWS::PLAN_TO_WATCH
];
public static function getIdToWatchingStatusMap() public static function getIdToWatchingStatusMap()
{ {

View File

@ -46,12 +46,6 @@ class ListItem {
->setFormFields($createData) ->setFormFields($createData)
->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password'])) ->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password']))
->getFullRequest(); ->getFullRequest();
/* $response = $this->getResponse('POST', "animelist/add/{$id}.xml", [
'body' => $this->fixBody((new FormBody)->addFields($createData))
]);
return $response->getBody() === 'Created'; */
} }
public function delete(string $id): Request public function delete(string $id): Request
@ -65,11 +59,7 @@ class ListItem {
->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password'])) ->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password']))
->getFullRequest(); ->getFullRequest();
/*$response = $this->getResponse('DELETE', "animelist/delete/{$id}.xml", [ // return $response->getBody() === 'Deleted'
'body' => $this->fixBody((new FormBody)->addField('id', $id))
]);
return $response->getBody() === 'Deleted';*/
} }
public function get(string $id): array public function get(string $id): array
@ -93,9 +83,5 @@ class ListItem {
]) ])
->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password'])) ->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password']))
->getFullRequest(); ->getFullRequest();
/* return $this->getResponse('POST', "animelist/update/{$id}.xml", [
'body' => $this->fixBody($body)
]); */
} }
} }

View File

@ -36,13 +36,18 @@ class Model {
protected $animeListTransformer; protected $animeListTransformer;
/** /**
* KitsuModel constructor. * MAL Model constructor.
*/ */
public function __construct(ListItem $listItem) public function __construct(ListItem $listItem)
{ {
$this->animeListTransformer = new AnimeListTransformer(); $this->animeListTransformer = new AnimeListTransformer();
$this->listItem = $listItem; $this->listItem = $listItem;
} }
public function createFullListItem(array $data): Request
{
return $this->listItem->create($data);
}
public function createListItem(array $data): Request public function createListItem(array $data): Request
{ {
@ -70,7 +75,7 @@ class Model {
] ]
]); ]);
return $list;//['anime']; return $list['myanimelist']['anime'];
} }
public function getListItem(string $listId): array public function getListItem(string $listId): array

View File

@ -32,6 +32,12 @@ class AnimeListTransformer extends AbstractTransformer {
AnimeWatchingStatus::PLAN_TO_WATCH => '6' AnimeWatchingStatus::PLAN_TO_WATCH => '6'
]; ];
/**
* Transform MAL episode data to Kitsu episode data
*
* @param array $item
* @return array
*/
public function transform($item) public function transform($item)
{ {
$rewatching = (array_key_exists('rewatching', $item) && $item['rewatching']); $rewatching = (array_key_exists('rewatching', $item) && $item['rewatching']);

View File

@ -1,33 +0,0 @@
<?php declare(strict_types=1);
/**
* Anime List Client
*
* An API client for Kitsu and MyAnimeList to manage anime and manga watch lists
*
* PHP version 7
*
* @package AnimeListClient
* @author Timothy J. Warren <tim@timshomepage.net>
* @copyright 2015 - 2017 Timothy J. Warren
* @license http://www.opensource.org/licenses/mit-license.html MIT License
* @version 4.0
* @link https://github.com/timw4mail/HummingBirdAnimeClient
*/
namespace Aviat\AnimeClient\API\MAL;
use Aviat\Ion\Transformer\AbstractTransformer;
class MALToKitsuTransformer extends AbstractTransformer {
public function transform($item)
{
}
public function untransform($item)
{
}
}

View File

@ -24,11 +24,10 @@ use Aviat\AnimeClient\{
Model, Model,
Util Util
}; };
use Aviat\AnimeClient\API\{ use Aviat\AnimeClient\API\CacheTrait;
CacheTrait, use Aviat\AnimeClient\API\{Kitsu, MAL};
Kitsu, use Aviat\AnimeClient\API\Kitsu\KitsuRequestBuilder;
MAL use Aviat\AnimeClient\API\MAL\MALRequestBuilder;
};
use Aviat\Banker\Pool; use Aviat\Banker\Pool;
use Aviat\Ion\Config; use Aviat\Ion\Config;
use Aviat\Ion\Di\{Container, ContainerAware}; use Aviat\Ion\Di\{Container, ContainerAware};
@ -109,21 +108,35 @@ class BaseCommand extends Command {
// Models // Models
$container->set('kitsu-model', function($container) { $container->set('kitsu-model', function($container) {
$requestBuilder = new KitsuRequestBuilder();
$requestBuilder->setLogger($container->getLogger('kitsu-request'));
$listItem = new Kitsu\ListItem(); $listItem = new Kitsu\ListItem();
$listItem->setContainer($container); $listItem->setContainer($container);
$listItem->setRequestBuilder($requestBuilder);
$model = new Kitsu\Model($listItem); $model = new Kitsu\Model($listItem);
$model->setContainer($container); $model->setContainer($container);
$model->setRequestBuilder($requestBuilder);
$cache = $container->get('cache'); $cache = $container->get('cache');
$model->setCache($cache); $model->setCache($cache);
return $model; return $model;
}); });
$container->set('mal-model', function($container) { $container->set('mal-model', function($container) {
$requestBuilder = new MALRequestBuilder();
$requestBuilder->setLogger($container->getLogger('mal-request'));
$listItem = new MAL\ListItem(); $listItem = new MAL\ListItem();
$listItem->setContainer($container); $listItem->setContainer($container);
$listItem->setRequestBuilder($requestBuilder);
$model = new MAL\Model($listItem); $model = new MAL\Model($listItem);
$model->setContainer($container); $model->setContainer($container);
$model->setRequestBuilder($requestBuilder);
return $model; return $model;
}); });
$container->set('util', function($container) { $container->set('util', function($container) {
return new Util($container); return new Util($container);
}); });

View File

@ -16,8 +16,13 @@
namespace Aviat\AnimeClient\Command; namespace Aviat\AnimeClient\Command;
use function Amp\{all, wait};
use Amp\Artax; use Amp\Artax;
use Aviat\AnimeClient\API\Kitsu; use Amp\Artax\Client;
use Aviat\AnimeClient\API\{JsonAPI, Kitsu, MAL};
use Aviat\AnimeClient\API\MAL\Transformer\AnimeListTransformer as ALT;
use Aviat\Ion\Json;
/** /**
* Clears the API Cache * Clears the API Cache
@ -41,91 +46,173 @@ class SyncKitsuWithMal extends BaseCommand {
$this->setCache($this->container->get('cache')); $this->setCache($this->container->get('cache'));
$this->kitsuModel = $this->container->get('kitsu-model'); $this->kitsuModel = $this->container->get('kitsu-model');
$this->malModel = $this->container->get('mal-model'); $this->malModel = $this->container->get('mal-model');
//$kitsuCount = $this->getKitsuAnimeListPageCount(); $malCount = count($this->getMALList());
//$this->echoBox("List item count: {$kitsuCount}"); $kitsuCount = $this->getKitsuAnimeListPageCount();
$this->MALItemCreate();
$this->echoBox("Number of MAL list items: {$malCount}");
//echo json_encode($this->getMALList(), \JSON_PRETTY_PRINT); $this->echoBox("Number of Kitsu list items: {$kitsuCount}");
$data = $this->diffLists();
$this->echoBox("Number of items that need to be added to MAL: " . count($data));
if (! empty($data['addToMAL']))
{
$this->echoBox("Adding missing list items to MAL");
$this->createMALListItems($data['addToMAL']);
}
} }
public function getKitsuList()
{
$count = $this->getKitsuAnimeListPageCount();
$size = 100;
$pages = ceil($count / $size);
$requests = [];
// Set up requests
for ($i = 0; $i < $count; $i++)
{
$offset = $i * $size;
$requests[] = $this->kitsuModel->getFullAnimeList($size, $offset);
}
$promiseArray = (new Client())->requestMulti($requests);
$responses = wait(all($promiseArray));
$output = [];
foreach($responses as $response)
{
$data = Json::decode($response->getBody());
$output = array_merge_recursive($output, $data);
}
return $output;
}
public function getMALList() public function getMALList()
{ {
return $this->malModel->getFullList(); return $this->malModel->getFullList();
} }
public function filterMappings(array $includes): array
{
$output = [];
foreach($includes as $id => $mapping)
{
if ($mapping['externalSite'] === 'myanimelist/anime')
{
$output[$id] = $mapping;
}
}
return $output;
}
// 2015-05-20T23:48:47.731Z
public function formatMALList()
{
$orig = $this->getMALList();
$output = [];
foreach($orig as $item)
{
$output[$item['series_animedb_id']] = [
'id' => $item['series_animedb_id'],
'data' => [
'status' => MAL::MAL_KITSU_WATCHING_STATUS_MAP[$item['my_status']],
'progress' => $item['my_watched_episodes'],
'reconsuming' => (bool) $item['my_rewatching'],
'reconsumeCount' => array_key_exists('times_rewatched', $item)
? $item['times_rewatched']
: 0,
// 'notes' => ,
'rating' => $item['my_score'],
'updatedAt' => (new \DateTime())
->setTimestamp((int)$item['my_last_updated'])
->format(\DateTime::W3C),
]
];
}
return $output;
}
public function filterKitsuList()
{
$data = $this->getKitsuList();
$includes = JsonAPI::organizeIncludes($data['included']);
$includes['mappings'] = $this->filterMappings($includes['mappings']);
$output = [];
foreach($data['data'] as $listItem)
{
$animeId = $listItem['relationships']['anime']['data']['id'];
$potentialMappings = $includes['anime'][$animeId]['relationships']['mappings'];
$malId = null;
foreach ($potentialMappings as $mappingId)
{
if (array_key_exists($mappingId, $includes['mappings']))
{
$malId = $includes['mappings'][$mappingId]['externalId'];
}
}
// Skip to the next item if there isn't a MAL ID
if ($malId === null)
{
continue;
}
$output[$listItem['id']] = [
'id' => $listItem['id'],
'malId' => $malId,
'data' => $listItem['attributes'],
];
}
return $output;
}
public function getKitsuAnimeListPageCount() public function getKitsuAnimeListPageCount()
{ {
$cacheItem = $this->cache->getItem(Kitsu::AUTH_TOKEN_CACHE_KEY); return $this->kitsuModel->getAnimeListCount();
$query = http_build_query([
'filter' => [
'user_id' => $this->kitsuModel->getUserIdByUsername(),
'media_type' => 'Anime'
],
// 'include' => 'anime,anime.genres,anime.mappings,anime.streamingLinks',
'page' => [
'limit' => 1
],
'sort' => '-updated_at'
]);
$request = (new Artax\Request)
->setUri("https://kitsu.io/api/edge/library-entries?{$query}")
->setProtocol('1.1')
->setAllHeaders([
'Accept' => 'application/vnd.api+json',
'Content-Type' => 'application/vnd.api+json',
'User-Agent' => "Tim's Anime Client/4.0"
]);
if ($cacheItem->isHit())
{
$token = $cacheItem->get();
$request->setHeader('Authorization', "bearer {$token}");
}
else
{
$this->echoBox("WARNING: NOT LOGGED IN\nSome data might be missing");
}
$response = \Amp\wait((new Artax\Client)->request($request));
$body = json_decode($response->getBody(), TRUE);
return $body['meta']['count'];
}
public function MALItemCreate()
{
$input = json_decode('{
"watching_status": "current",
"user_rating": "",
"episodes_watched": "4",
"rewatched": "0",
"notes": "",
"id": "15794526",
"mal_id": "33731",
"edit": "true"
}', TRUE);
$response = $this->malModel->createListItem([
'id' => 12255,
'status' => 'planned',
'type' => 'anime'
]);
//$response = $this->malModel->updateListItem($input);
//print_r($response);
//echo $response->getBody();
} }
public function diffLists() public function diffLists()
{ {
// Get libraryEntries with media.mappings from Kitsu // Get libraryEntries with media.mappings from Kitsu
// Organize mappings, and ignore entries without mappings // Organize mappings, and ignore entries without mappings
$kitsuList = $this->filterKitsuList();
// Get MAL list data // Get MAL list data
$malList = $this->formatMALList();
$itemsToAddToMAL = [];
foreach($kitsuList as $kitsuItem)
{
if (array_key_exists($kitsuItem['malId'], $malList))
{
// Eventually, compare the list entries, and determine which
// needs to be updated
continue;
}
// Looks like this item only exists on Kitsu
$itemsToAddToMAL[] = [
'mal_id' => $kitsuItem['malId'],
'data' => $kitsuItem['data']
];
}
// Compare each list entry // Compare each list entry
// If a list item exists only on MAL, create it on Kitsu with the existing data from MAL // If a list item exists only on MAL, create it on Kitsu with the existing data from MAL
@ -135,7 +222,39 @@ class SyncKitsuWithMal extends BaseCommand {
// Otherwise, use rewatch count, then episode progress as critera for selecting the more up // Otherwise, use rewatch count, then episode progress as critera for selecting the more up
// to date entry // to date entry
// Based on the 'newer' entry, update the other api list item // Based on the 'newer' entry, update the other api list item
return [
'addToMAL' => $itemsToAddToMAL,
];
} }
public function createMALListItems($itemsToAdd)
{
$transformer = new ALT();
$requests = [];
foreach($itemsToAdd as $item)
{
$data = $transformer->untransform($item);
$requests[] = $this->malModel->createFullListItem($data);
}
$promiseArray = (new Client())->requestMulti($requests);
$responses = wait(all($promiseArray));
foreach($responses as $key => $response)
{
$id = $itemsToAdd[$key]['mal_id'];
if ($response->getBody() === 'Created')
{
$this->echoBox("Successfully create list item with id: {$id}");
}
else
{
$this->echoBox("Failed to create list item with id: {$id}");
}
}
}
} }