From ecb913322f5017f6bf2f0aea8a4451247c6b664c Mon Sep 17 00:00:00 2001 From: "Timothy J. Warren" Date: Fri, 9 Oct 2020 16:18:45 -0400 Subject: [PATCH] Pull anime lists from GraphQL, see #33 --- src/AnimeClient/API/Kitsu/AnimeTrait.php | 48 +++++- src/AnimeClient/API/Kitsu/Model.php | 83 +++++++++- .../API/Kitsu/Queries/GetLibrary.graphql | 3 + src/AnimeClient/API/Kitsu/RequestBuilder.php | 63 +++---- .../Transformer/AnimeListTransformer.php | 49 +++--- .../Transformer/OldAnimeListTransformer.php | 156 ++++++++++++++++++ src/AnimeClient/AnimeClient.php | 1 - src/AnimeClient/Controller/Anime.php | 4 +- .../Transformer/AnimeListTransformerTest.php | 4 +- 9 files changed, 340 insertions(+), 71 deletions(-) create mode 100644 src/AnimeClient/API/Kitsu/Transformer/OldAnimeListTransformer.php diff --git a/src/AnimeClient/API/Kitsu/AnimeTrait.php b/src/AnimeClient/API/Kitsu/AnimeTrait.php index dc50309b..2c6a20cf 100644 --- a/src/AnimeClient/API/Kitsu/AnimeTrait.php +++ b/src/AnimeClient/API/Kitsu/AnimeTrait.php @@ -17,11 +17,12 @@ namespace Aviat\AnimeClient\API\Kitsu; use Amp\Http\Client\Request; +use Aviat\AnimeClient\API\Kitsu\Transformer\AnimeListTransformer; use Aviat\AnimeClient\Kitsu as K; use Aviat\AnimeClient\API\Enum\AnimeWatchingStatus\Kitsu as KitsuWatchingStatus; use Aviat\AnimeClient\API\JsonAPI; use Aviat\AnimeClient\API\Kitsu\Transformer\AnimeHistoryTransformer; -use Aviat\AnimeClient\API\Kitsu\Transformer\AnimeListTransformer; +use Aviat\AnimeClient\API\Kitsu\Transformer\OldAnimeListTransformer; use Aviat\AnimeClient\API\Kitsu\Transformer\AnimeTransformer; use Aviat\AnimeClient\API\Mapping\AnimeWatchingStatus; use Aviat\AnimeClient\API\ParallelAPIRequest; @@ -39,9 +40,9 @@ trait AnimeTrait { * to a common format used by * templates * - * @var AnimeListTransformer + * @var OldAnimeListTransformer */ - protected AnimeListTransformer $animeListTransformer; + protected OldAnimeListTransformer $oldListTransformer; /** * @var AnimeTransformer @@ -124,6 +125,45 @@ trait AnimeTrait { $list = $this->cache->get($key, NULL); + if ($list === NULL) + { + $data = $this->getRawList(ListType::ANIME, $status) ?? []; + + // Bail out on no data + if (empty($data)) + { + return []; + } + + $transformer = new AnimeListTransformer(); + $transformed = $transformer->transformCollection($data); + $keyed = []; + + foreach($transformed as $item) + { + $keyed[$item['id']] = $item; + } + + $list = $keyed; + $this->cache->set($key, $list); + } + + return $list; + } + + /** + * Get the anime list for the configured user + * + * @param string $status - The watching status to filter the list with + * @return array + * @throws InvalidArgumentException + */ + public function oldGetAnimeList(string $status): array + { + $key = "kitsu-anime-list-{$status}"; + + $list = $this->cache->get($key, NULL); + if ($list === NULL) { $data = $this->getRawAnimeList($status) ?? []; @@ -142,7 +182,7 @@ trait AnimeTrait { $item['included'] = $included; } unset($item); - $transformed = $this->animeListTransformer->transformCollection($data['data']); + $transformed = $this->oldListTransformer->transformCollection($data['data']); $keyed = []; foreach($transformed as $item) diff --git a/src/AnimeClient/API/Kitsu/Model.php b/src/AnimeClient/API/Kitsu/Model.php index 89db3804..4db7c866 100644 --- a/src/AnimeClient/API/Kitsu/Model.php +++ b/src/AnimeClient/API/Kitsu/Model.php @@ -17,7 +17,9 @@ namespace Aviat\AnimeClient\API\Kitsu; use function Amp\Promise\wait; +use function Aviat\AnimeClient\getApiClient; +use Amp; use Amp\Http\Client\Request; use Aviat\AnimeClient\Kitsu as K; use Aviat\AnimeClient\API\{ @@ -27,7 +29,7 @@ use Aviat\AnimeClient\API\{ }; use Aviat\AnimeClient\API\Kitsu\Transformer\{ AnimeTransformer, - AnimeListTransformer, + OldAnimeListTransformer, LibraryEntryTransformer, MangaTransformer, MangaListTransformer @@ -49,7 +51,7 @@ final class Model { use MangaTrait; use MutationTrait; - protected const LIST_PAGE_SIZE = 75; + protected const LIST_PAGE_SIZE = 100; /** * @var ListItem @@ -64,7 +66,7 @@ final class Model { public function __construct(ListItem $listItem) { $this->animeTransformer = new AnimeTransformer(); - $this->animeListTransformer = new AnimeListTransformer(); + $this->oldListTransformer = new OldAnimeListTransformer(); $this->mangaTransformer = new MangaTransformer(); $this->mangaListTransformer = new MangaListTransformer(); @@ -351,6 +353,81 @@ final class Model { ]); } + /** + * Get the raw anime/manga list from GraphQL + * + * @param string $type + * @param string $status + * @return array + */ + public function getRawList(string $type, string $status = ''): array + { + $pages = []; + + foreach ($this->getRawListPages(strtoupper($type), strtoupper($status)) as $page) + { + $pages[] = $page; + } + + return array_merge(...$pages); + } + + protected function getRawListPages(string $type, string $status = ''): ?\Generator + { + $items = $this->getRawPages($type, $status); + + while (wait($items->advance())) + { + yield $items->getCurrent(); + } + } + + protected function getRawPages(string $type, string $status = ''): Amp\Iterator + { + $cursor = ''; + $username = $this->getUsername(); + + return new Amp\Producer(function (callable $emit) use ($type, $status, $cursor, $username) { + while (TRUE) + { + $vars = [ + 'type' => $type, + 'slug' => $username, + ]; + if ($status !== '') + { + $vars['status'] = $status; + } + if ($cursor !== '') + { + $vars['after'] = $cursor; + } + + $request = $this->requestBuilder->queryRequest('GetLibrary', $vars); + $response = yield getApiClient()->request($request); + $json = yield $response->getBody()->buffer(); + + $rawData = Json::decode($json); + $data = $rawData['data']['findProfileBySlug']['library']['all'] ?? []; + $page = $data['pageInfo']; + if (empty($data)) + { + // @TODO Error logging + break; + } + + $cursor = $page['endCursor']; + + yield $emit($data['nodes']); + + if ($page['hasNextPage'] === FALSE) + { + break; + } + } + }); + } + private function getUserId(): string { static $userId = NULL; diff --git a/src/AnimeClient/API/Kitsu/Queries/GetLibrary.graphql b/src/AnimeClient/API/Kitsu/Queries/GetLibrary.graphql index cf87b320..e28e44a5 100644 --- a/src/AnimeClient/API/Kitsu/Queries/GetLibrary.graphql +++ b/src/AnimeClient/API/Kitsu/Queries/GetLibrary.graphql @@ -52,6 +52,8 @@ query ( sfw slug status + startDate + endDate type titles { canonical @@ -60,6 +62,7 @@ query ( } ...on Anime { episodeCount + episodeLength streamingLinks(first: 10) { nodes { dubs diff --git a/src/AnimeClient/API/Kitsu/RequestBuilder.php b/src/AnimeClient/API/Kitsu/RequestBuilder.php index e103c42a..2afb4d6c 100644 --- a/src/AnimeClient/API/Kitsu/RequestBuilder.php +++ b/src/AnimeClient/API/Kitsu/RequestBuilder.php @@ -180,14 +180,7 @@ final class RequestBuilder extends APIRequestBuilder { return ($response->getStatus() === 204); } - /** - * Run a GraphQL API query - * - * @param string $name - * @param array $variables - * @return array - */ - public function runQuery(string $name, array $variables = []): array + public function queryRequest(string $name, array $variables = []): Request { $file = __DIR__ . "/Queries/{$name}.graphql"; if ( ! file_exists($file)) @@ -209,11 +202,33 @@ final class RequestBuilder extends APIRequestBuilder { } } - return $this->graphResponse([ - 'body' => $body + return $this->setUpRequest('POST', K::GRAPHQL_ENDPOINT, [ + 'body' => $body, ]); } + /** + * Run a GraphQL API query + * + * @param string $name + * @param array $variables + * @return array + */ + public function runQuery(string $name, array $variables = []): array + { + $request = $this->queryRequest($name, $variables); + $response = getResponse($request); + $validResponseCodes = [200, 201]; + + if ( ! \in_array($response->getStatus(), $validResponseCodes, TRUE)) + { + $logger = $this->container->getLogger('kitsu-graphql'); + $logger->warning('Non 200 response for GraphQL call', (array)$response->getBody()); + } + + return Json::decode(wait($response->getBody()->buffer())); + } + /** * @param string $name * @param array $variables @@ -256,6 +271,13 @@ final class RequestBuilder extends APIRequestBuilder { { $request = $this->mutateRequest($name, $variables); $response = getResponse($request); + $validResponseCodes = [200, 201]; + + if ( ! \in_array($response->getStatus(), $validResponseCodes, TRUE)) + { + $logger = $this->container->getLogger('kitsu-graphql'); + $logger->warning('Non 200 response for GraphQL call', (array)$response->getBody()); + } return Json::decode(wait($response->getBody()->buffer())); } @@ -286,27 +308,6 @@ final class RequestBuilder extends APIRequestBuilder { return $response; } - /** - * Remove some boilerplate for GraphQL requests - * - * @param array $options - * @return array - * @throws \Throwable - */ - protected function graphResponse(array $options = []): array - { - $response = $this->getResponse('POST', K::GRAPHQL_ENDPOINT, $options); - $validResponseCodes = [200, 201]; - - if ( ! \in_array($response->getStatus(), $validResponseCodes, TRUE)) - { - $logger = $this->container->getLogger('kitsu-graphql'); - $logger->warning('Non 200 response for GraphQL call', (array)$response->getBody()); - } - - return Json::decode(wait($response->getBody()->buffer())); - } - /** * Make a request * diff --git a/src/AnimeClient/API/Kitsu/Transformer/AnimeListTransformer.php b/src/AnimeClient/API/Kitsu/Transformer/AnimeListTransformer.php index 078b8105..4a2c0e40 100644 --- a/src/AnimeClient/API/Kitsu/Transformer/AnimeListTransformer.php +++ b/src/AnimeClient/API/Kitsu/Transformer/AnimeListTransformer.php @@ -38,34 +38,27 @@ final class AnimeListTransformer extends AbstractTransformer { */ public function transform($item): AnimeListItem { - $included = $item['included']; - $animeId = $item['relationships']['media']['data']['id']; - $anime = $included['anime'][$animeId]; + $animeId = $item['media']['id']; + $anime = $item['media']; $genres = []; - foreach($anime['relationships']['categories'] as $genre) - { - $genres[] = $genre['title']; - } - - sort($genres); - - $rating = (int) $item['attributes']['ratingTwenty'] !== 0 - ? $item['attributes']['ratingTwenty'] / 2 + $rating = (int) $item['rating'] !== 0 + ? (int)$item['rating'] / 2 : '-'; - $total_episodes = array_key_exists('episodeCount', $anime) && (int) $anime['episodeCount'] !== 0 + $total_episodes = (int) $anime['episodeCount'] !== 0 ? (int) $anime['episodeCount'] : '-'; $MALid = NULL; - if (array_key_exists('mappings', $anime['relationships'])) + $mappings = $anime['mappings']['nodes'] ?? []; + if ( ! empty($mappings)) { - foreach ($anime['relationships']['mappings'] as $mapping) + foreach ($mappings as $mapping) { - if ($mapping['externalSite'] === 'myanimelist/anime') + if ($mapping['externalSite'] === 'MYANIMELIST_ANIME') { $MALid = $mapping['externalId']; break; @@ -73,19 +66,19 @@ final class AnimeListTransformer extends AbstractTransformer { } } - $streamingLinks = array_key_exists('streamingLinks', $anime['relationships']) - ? Kitsu::parseListItemStreamingLinks($included, $animeId) + $streamingLinks = array_key_exists('nodes', $anime['streamingLinks']) + ? Kitsu::parseStreamingLinks($anime['streamingLinks']['nodes']) : []; - $titles = Kitsu::filterTitles($anime); - $title = array_shift($titles); + $titles = Kitsu::getFilteredTitles($anime['titles']); + $title = $anime['titles']['canonical']; return AnimeListItem::from([ 'id' => $item['id'], 'mal_id' => $MALid, 'episodes' => [ - 'watched' => (int) $item['attributes']['progress'] !== 0 - ? (int) $item['attributes']['progress'] + 'watched' => (int) $item['progress'] !== 0 + ? (int) $item['progress'] : '-', 'total' => $total_episodes, 'length' => $anime['episodeLength'], @@ -102,16 +95,16 @@ final class AnimeListTransformer extends AbstractTransformer { 'titles' => $titles, 'slug' => $anime['slug'], 'show_type' => (string)StringType::from($anime['subtype'])->upperCaseFirst(), - 'cover_image' => $anime['posterImage']['small'], + 'cover_image' => $anime['posterImage']['views'][1]['url'], 'genres' => $genres, 'streaming_links' => $streamingLinks, ], - 'watching_status' => $item['attributes']['status'], - 'notes' => $item['attributes']['notes'], - 'rewatching' => (bool) $item['attributes']['reconsuming'], - 'rewatched' => (int) $item['attributes']['reconsumeCount'], + 'watching_status' => $item['status'], + 'notes' => $item['notes'], + 'rewatching' => (bool) $item['reconsuming'], + 'rewatched' => (int) $item['reconsumeCount'], 'user_rating' => $rating, - 'private' => $item['attributes']['private'] ?? FALSE, + 'private' => $item['private'] ?? FALSE, ]); } diff --git a/src/AnimeClient/API/Kitsu/Transformer/OldAnimeListTransformer.php b/src/AnimeClient/API/Kitsu/Transformer/OldAnimeListTransformer.php new file mode 100644 index 00000000..0268eb16 --- /dev/null +++ b/src/AnimeClient/API/Kitsu/Transformer/OldAnimeListTransformer.php @@ -0,0 +1,156 @@ + + * @copyright 2015 - 2020 Timothy J. Warren + * @license http://www.opensource.org/licenses/mit-license.html MIT License + * @version 5.1 + * @link https://git.timshomepage.net/timw4mail/HummingBirdAnimeClient + */ + +namespace Aviat\AnimeClient\API\Kitsu\Transformer; + +use Aviat\AnimeClient\Kitsu; +use Aviat\AnimeClient\Types\{ + FormItem, + AnimeListItem +}; +use Aviat\Ion\Transformer\AbstractTransformer; +use Aviat\Ion\Type\StringType; + +/** + * Transformer for anime list + */ +final class OldAnimeListTransformer extends AbstractTransformer { + + /** + * Convert raw api response to a more + * logical and workable structure + * + * @param array $item API library item + * @return AnimeListItem + */ + public function transform($item): AnimeListItem + { + $included = $item['included']; + $animeId = $item['relationships']['media']['data']['id']; + $anime = $included['anime'][$animeId]; + + $genres = []; + + foreach($anime['relationships']['categories'] as $genre) + { + $genres[] = $genre['title']; + } + + sort($genres); + + $rating = (int) $item['attributes']['ratingTwenty'] !== 0 + ? $item['attributes']['ratingTwenty'] / 2 + : '-'; + + $total_episodes = array_key_exists('episodeCount', $anime) && (int) $anime['episodeCount'] !== 0 + ? (int) $anime['episodeCount'] + : '-'; + + $MALid = NULL; + + if (array_key_exists('mappings', $anime['relationships'])) + { + foreach ($anime['relationships']['mappings'] as $mapping) + { + if ($mapping['externalSite'] === 'myanimelist/anime') + { + $MALid = $mapping['externalId']; + break; + } + } + } + + $streamingLinks = array_key_exists('streamingLinks', $anime['relationships']) + ? Kitsu::parseListItemStreamingLinks($included, $animeId) + : []; + + $titles = Kitsu::filterTitles($anime); + $title = array_shift($titles); + + return AnimeListItem::from([ + 'id' => $item['id'], + 'mal_id' => $MALid, + 'episodes' => [ + 'watched' => (int) $item['attributes']['progress'] !== 0 + ? (int) $item['attributes']['progress'] + : '-', + 'total' => $total_episodes, + 'length' => $anime['episodeLength'], + ], + 'airing' => [ + 'status' => Kitsu::getAiringStatus($anime['startDate'], $anime['endDate']), + 'started' => $anime['startDate'], + 'ended' => $anime['endDate'] + ], + 'anime' => [ + 'id' => $animeId, + 'age_rating' => $anime['ageRating'], + 'title' => $title, + 'titles' => $titles, + 'slug' => $anime['slug'], + 'show_type' => (string)StringType::from($anime['subtype'])->upperCaseFirst(), + 'cover_image' => $anime['posterImage']['small'], + 'genres' => $genres, + 'streaming_links' => $streamingLinks, + ], + 'watching_status' => $item['attributes']['status'], + 'notes' => $item['attributes']['notes'], + 'rewatching' => (bool) $item['attributes']['reconsuming'], + 'rewatched' => (int) $item['attributes']['reconsumeCount'], + 'user_rating' => $rating, + 'private' => $item['attributes']['private'] ?? FALSE, + ]); + } + + /** + * Convert transformed data to + * api response format + * + * @param array $item Transformed library item + * @return FormItem API library item + */ + public function untransform($item): FormItem + { + $privacy = (array_key_exists('private', $item) && $item['private']); + $rewatching = (array_key_exists('rewatching', $item) && $item['rewatching']); + + $untransformed = FormItem::from([ + 'id' => $item['id'], + 'anilist_item_id' => $item['anilist_item_id'] ?? NULL, + 'mal_id' => $item['mal_id'] ?? NULL, + 'data' => [ + 'status' => $item['watching_status'], + 'reconsuming' => $rewatching, + 'reconsumeCount' => $item['rewatched'], + 'notes' => $item['notes'], + 'private' => $privacy + ] + ]); + + if (is_numeric($item['episodes_watched']) && $item['episodes_watched'] > 0) + { + $untransformed['data']['progress'] = (int) $item['episodes_watched']; + } + + if (is_numeric($item['user_rating']) && $item['user_rating'] > 0) + { + $untransformed['data']['ratingTwenty'] = $item['user_rating'] * 2; + } + + return $untransformed; + } +} +// End of AnimeListTransformer.php \ No newline at end of file diff --git a/src/AnimeClient/AnimeClient.php b/src/AnimeClient/AnimeClient.php index c9c70216..3aa76ffe 100644 --- a/src/AnimeClient/AnimeClient.php +++ b/src/AnimeClient/AnimeClient.php @@ -240,7 +240,6 @@ function getResponse ($request): Response $request = new Request($request); } - return wait($client->request($request)); } diff --git a/src/AnimeClient/Controller/Anime.php b/src/AnimeClient/Controller/Anime.php index 41326edc..1a5e17dc 100644 --- a/src/AnimeClient/Controller/Anime.php +++ b/src/AnimeClient/Controller/Anime.php @@ -18,7 +18,7 @@ namespace Aviat\AnimeClient\Controller; use Aura\Router\Exception\RouteNotFound; use Aviat\AnimeClient\Controller as BaseController; -use Aviat\AnimeClient\API\Kitsu\Transformer\AnimeListTransformer; +use Aviat\AnimeClient\API\Kitsu\Transformer\OldAnimeListTransformer; use Aviat\AnimeClient\API\Enum\AnimeWatchingStatus\Kitsu as KitsuWatchingStatus; use Aviat\AnimeClient\API\Mapping\AnimeWatchingStatus; use Aviat\AnimeClient\Model\Anime as AnimeModel; @@ -228,7 +228,7 @@ final class Anime extends BaseController { // Do some minor data manipulation for // large form-based updates - $transformer = new AnimeListTransformer(); + $transformer = new OldAnimeListTransformer(); $postData = $transformer->untransform($data); $fullResult = $this->model->updateLibraryItem(FormItem::from($postData)); diff --git a/tests/AnimeClient/API/Kitsu/Transformer/AnimeListTransformerTest.php b/tests/AnimeClient/API/Kitsu/Transformer/AnimeListTransformerTest.php index ad742a8a..bdb1f81f 100644 --- a/tests/AnimeClient/API/Kitsu/Transformer/AnimeListTransformerTest.php +++ b/tests/AnimeClient/API/Kitsu/Transformer/AnimeListTransformerTest.php @@ -16,7 +16,7 @@ namespace Aviat\AnimeClient\Tests\API\Kitsu\Transformer; -use Aviat\AnimeClient\API\Kitsu\Transformer\AnimeListTransformer; +use Aviat\AnimeClient\API\Kitsu\Transformer\OldAnimeListTransformer; use Aviat\AnimeClient\Tests\AnimeClientTestCase; use Aviat\Ion\Friend; use Aviat\Ion\Json; @@ -33,7 +33,7 @@ class AnimeListTransformerTest extends AnimeClientTestCase { $this->beforeTransform = Json::decodeFile("{$this->dir}/animeListItemBeforeTransform.json"); - $this->transformer = new AnimeListTransformer(); + $this->transformer = new OldAnimeListTransformer(); } public function testTransform()