Sync Kitsu and Anilist both via GraphQL, see #33

src/AnimeClient/API/Kitsu/MangaTrait.php

@@ -19,7 +19,6 @@ namespace Aviat\AnimeClient\API\Kitsu;
 use Amp\Http\Client\Request;
 use Aviat\AnimeClient\Kitsu as K;
 use Aviat\AnimeClient\API\Enum\MangaReadingStatus\Kitsu as KitsuReadingStatus;
-use Aviat\AnimeClient\API\JsonAPI;
 use Aviat\AnimeClient\API\Kitsu\Transformer\MangaHistoryTransformer;
 use Aviat\AnimeClient\API\Kitsu\Transformer\MangaListTransformer;
 use Aviat\AnimeClient\API\Kitsu\Transformer\OldMangaListTransformer;

src/AnimeClient/API/Kitsu/Model.php

@@ -329,16 +329,27 @@ final class Model {
 	 */
 	public function getSyncList(string $type): array
 	{
-		$options = [
+		$statuses = [
-			'filter' => [
+			'CURRENT',
-				'user_id' => $this->getUserId(),
+			'PLANNED',
-				'kind' => $type,
+			'ON_HOLD',
-			],
+			'DROPPED',
-			'include' => "{$type},{$type}.mappings",
+			'COMPLETED',
-			// 'sort' => '-updated_at'
 		];
 
-		return $this->getRawSyncList($type, $options);
+		$pages = [];
+
+		// Although I can fetch the whole list without segregating by status,
+		// this way is much faster...
+		foreach ($statuses as $status)
+		{
+			foreach ($this->getRawSyncListPages(strtoupper($type), $status) as $page)
+			{
+				$pages[] = $page;
+			}
+		}
+
+		return array_merge(...$pages);
 	}
 
 	/**
@@ -412,6 +423,9 @@ final class Model {
 				$page = $data['pageInfo'];
 				if (empty($data))
 				{
+					dump($rawData);
+					die();
+
 					// @TODO Error logging
 					break;
 				}
@@ -428,6 +442,58 @@ final class Model {
 		});
 	}
 
+	protected function getRawSyncListPages(string $type, string $status): ?\Generator
+	{
+		$items = $this->getRawSyncPages($type, $status);
+
+		while (wait($items->advance()))
+		{
+			yield $items->getCurrent();
+		}
+	}
+
+	protected function getRawSyncPages(string $type, $status): Amp\Iterator {
+		$cursor = '';
+		$username = $this->getUsername();
+
+		return new Amp\Producer(function (callable $emit) use ($type, $status, $cursor, $username) {
+			while (TRUE)
+			{
+				$vars = [
+					'type' => $type,
+					'slug' => $username,
+					'status' => $status,
+				];
+				if ($cursor !== '')
+				{
+					$vars['after'] = $cursor;
+				}
+
+				$request = $this->requestBuilder->queryRequest('GetSyncLibrary', $vars);
+				$response = yield getApiClient()->request($request);
+				$json = yield $response->getBody()->buffer();
+
+				$rawData = Json::decode($json);
+				$data = $rawData['data']['findProfileBySlug']['library']['all'] ?? [];
+				$page = $data['pageInfo'];
+				if (empty($data))
+				{
+					dump($rawData);
+					die();
+				}
+
+				$cursor = $page['endCursor'];
+
+				yield $emit($data['nodes']);
+
+				if ($page['hasNextPage'] === FALSE)
+				{
+					break;
+				}
+			}
+		});
+	}
+
 	private function getUserId(): string
 	{
 		static $userId = NULL;
@@ -467,68 +533,4 @@ final class Model {
 
 		return $res['data']['findProfileBySlug']['library']['all']['totalCount'];
 	}
-
-	/**
-	 * Get the full anime list
-	 *
-	 * @param string $type
-	 * @param array $options
-	 * @return array
-	 * @throws InvalidArgumentException
-	 * @throws Throwable
-	 */
-	private function getRawSyncList(string $type, array $options): array
-	{
-		$count = $this->getListCount($type);
-		$size = static::LIST_PAGE_SIZE;
-		$pages = ceil($count / $size);
-
-		$requester = new ParallelAPIRequest();
-
-		// Set up requests
-		for ($i = 0; $i < $pages; $i++)
-		{
-			$offset = $i * $size;
-			$requester->addRequest($this->getRawSyncListPage($type, $size, $offset, $options));
-		}
-
-		$responses = $requester->makeRequests();
-		$output = [];
-
-		foreach($responses as $response)
-		{
-			$data = Json::decode($response);
-			$output[] = $data;
-		}
-
-		return array_merge_recursive(...$output);
-	}
-
-	/**
-	 * Get the full anime list in paginated form
-	 *
-	 * @param string $type
-	 * @param int $limit
-	 * @param int $offset
-	 * @param array $options
-	 * @return Request
-	 * @throws InvalidArgumentException
-	 */
-	private function getRawSyncListPage(string $type, int $limit, int $offset = 0, array $options = []): Request
-	{
-		$defaultOptions = [
-			'filter' => [
-				'user_id' => $this->getUserId(),
-				'kind' => $type,
-			],
-			'page' => [
-				'offset' => $offset,
-				'limit' => $limit
-			],
-			'sort' => '-updated_at'
-		];
-		$options = array_merge($defaultOptions, $options);
-
-		return $this->requestBuilder->setUpRequest('GET', 'library-entries', ['query' => $options]);
-	}
 }

src/AnimeClient/API/Kitsu/Queries/GetSyncLibrary.graphql

@@ -0,0 +1,42 @@
+query (
+	$slug: String!,
+	$type: MediaTypeEnum!,
+	$status: [LibraryEntryStatusEnum!],
+	$after: String
+) {
+	findProfileBySlug(slug: $slug) {
+		library {
+			all(first: 100, after: $after, mediaType: $type, status: $status) {
+				pageInfo {
+					endCursor
+					hasNextPage
+					hasPreviousPage
+					startCursor
+				}
+				totalCount
+				nodes {
+					id
+					notes
+					nsfw
+					private
+					progress
+					progressedAt
+					rating
+					reconsumeCount
+					reconsuming
+					status
+					media {
+						id
+						slug
+						mappings(first: 10) {
+							nodes {
+								externalId
+								externalSite
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+}

src/AnimeClient/Command/SyncLists.php

@@ -20,14 +20,12 @@ use ConsoleKit\Widgets;
 
 use Aviat\AnimeClient\API\{
 	Anilist\MissingIdException,
-	FailedResponseException,
-	JsonAPI,
 	ParallelAPIRequest
 };
 use Aviat\AnimeClient\API;
 use Aviat\AnimeClient\API\Anilist;
 use Aviat\AnimeClient\API\Mapping\{AnimeWatchingStatus, MangaReadingStatus};
-use Aviat\AnimeClient\Enum\{APISource, ListType, SyncAction};
+use Aviat\AnimeClient\Enum\{ListType, SyncAction};
 use Aviat\AnimeClient\Types\FormItem;
 use Aviat\Ion\Di\Exception\ContainerException;
 use Aviat\Ion\Di\Exception\NotFoundException;
@@ -40,6 +38,9 @@ use function in_array;
  * Syncs list data between Anilist and Kitsu
  */
 final class SyncLists extends BaseCommand {
+	protected const KITSU_GREATER = 1;
+	protected const ANILIST_GREATER = -1;
+	protected const SAME = 0;
 
 	/**
 	 * Model for making requests to Anilist API
@@ -315,37 +316,24 @@ final class SyncLists extends BaseCommand {
 			return [];
 		}
 
-		if ( ! array_key_exists('included', $data))
-		{
-			dump([
-				'problem' => 'Missing included data in method ' . __METHOD__,
-				'data' => $data
-			]);
-			return [];
-		}
-
-		$includes = JsonAPI::organizeIncludes($data['included']);
-		$includes['mappings'] = $this->filterMappings($includes['mappings'], $type);
-
 		$output = [];
 
-		foreach($data['data'] as $listItem)
+		foreach($data as $listItem)
 		{
-			$id = $listItem['relationships'][$type]['data']['id'];
+			// If there's no mapping, we can't sync, so continue
-
+			if ( ! is_array($listItem['media']['mappings']['nodes']))
-			$potentialMappings = $includes[$type][$id]['relationships']['mappings'];
+			{
+				continue;
+			}
 			$malId = NULL;
 
-			foreach ($potentialMappings as $mappingId)
+			foreach ($listItem['media']['mappings']['nodes'] as $mapping)
 			{
-				if (is_array($mappingId))
+				$uType = strtoupper($type);
+				if ($mapping['externalSite'] === "MYANIMELIST_{$uType}")
 				{
-					continue;
+					$malId = $mapping['externalId'];
-				}
+					break;
-
-				if (array_key_exists($mappingId, $includes['mappings']))
-				{
-					$malId = $includes['mappings'][$mappingId]['externalId'];
 				}
 			}
 
@@ -355,10 +343,22 @@ final class SyncLists extends BaseCommand {
 				continue;
 			}
 
-			$output[$listItem['id']] = [
+			$output[$listItem['media']['id']] = [
-				'id' => $listItem['id'],
+				'id' => $listItem['media']['id'],
+				'slug' => $listItem['media']['slug'],
 				'malId' => $malId,
-				'data' => $listItem['attributes'],
+				'data' => [
+					'notes' => $listItem['notes'],
+					'private' => $listItem['private'],
+					'progress' => $listItem['progress'],
+					// Comparision is done on 1-10 scale,
+					// Kitsu returns 1-20 scale.
+					'rating' => $listItem['rating'] / 2,
+					'reconsumeCount' => $listItem['reconsumeCount'],
+					'reconsuming' => $listItem['reconsuming'],
+					'status' => strtolower($listItem['status']),
+					'updatedAt' => $listItem['progressedAt'],
+				]
 			];
 		}
 
@@ -460,7 +460,7 @@ final class SyncLists extends BaseCommand {
 					'private' => $kItem['private'],
 					'progress' => $kItem['progress'],
 					'repeat' => $kItem['reconsumeCount'],
-					'score' => $kItem['ratingTwenty'] * 5, // 100 point score on Anilist
+					'score' => $kItem['rating'] * 10, // 100 point score on Anilist
 					'status' => $newItemStatus,
 				],
 			];
@@ -492,10 +492,9 @@ final class SyncLists extends BaseCommand {
 			'status',
 		];
 		$diff = [];
-		$dateDiff = new DateTime($kitsuItem['data']['updatedAt']) <=> new DateTime((string)$anilistItem['data']['updatedAt']);
+		$dateDiff = ($kitsuItem['data']['updatedAt'] !== NULL)
-
+			? new DateTime($kitsuItem['data']['updatedAt']) <=> new DateTime((string)$anilistItem['data']['updatedAt'])
-		// Correct differences in notation
+			: 0;
-		$kitsuItem['data']['rating'] = $kitsuItem['data']['ratingTwenty'] / 2;
 
 		foreach($compareKeys as $key)
 		{
@@ -535,12 +534,12 @@ final class SyncLists extends BaseCommand {
 		// If status is the same, and progress count is different, use greater progress
 		if ($sameStatus && ( ! $sameProgress))
 		{
-			if ($diff['progress'] === 1)
+			if ($diff['progress'] === self::KITSU_GREATER)
 			{
 				$update['data']['progress'] = $kitsuItem['data']['progress'];
 				$return['updateType'][] = API::ANILIST;
 			}
-			else if($diff['progress'] === -1)
+			else if($diff['progress'] === self::ANILIST_GREATER)
 			{
 				$update['data']['progress'] = $anilistItem['data']['progress'];
 				$return['updateType'][] = API::KITSU;
@@ -550,12 +549,12 @@ final class SyncLists extends BaseCommand {
 		// If status is different, use the status of the more recently updated item
 		if ( ! $sameStatus)
 		{
-			if ($dateDiff === 1)
+			if ($dateDiff === self::KITSU_GREATER)
 			{
 				$update['data']['status'] = $kitsuItem['data']['status'];
 				$return['updateType'][] = API::ANILIST;
 			}
-			else if ($dateDiff === -1)
+			else if ($dateDiff === self::ANILIST_GREATER)
 			{
 				$update['data']['status'] = $anilistItem['data']['status'];
 				$return['updateType'][] = API::KITSU;
@@ -566,7 +565,7 @@ final class SyncLists extends BaseCommand {
 		// But, at least for now, assume newer record is correct
 		if ( ! ($sameStatus || $sameProgress))
 		{
-			if ($dateDiff === 1)
+			if ($dateDiff === self::KITSU_GREATER)
 			{
 				$update['data']['status'] = $kitsuItem['data']['status'];
 
@@ -577,7 +576,7 @@ final class SyncLists extends BaseCommand {
 
 				$return['updateType'][] = API::ANILIST;
 			}
-			else if($dateDiff === -1)
+			else if($dateDiff === self::ANILIST_GREATER)
 			{
 				$update['data']['status'] = $anilistItem['data']['status'];
 
@@ -594,7 +593,7 @@ final class SyncLists extends BaseCommand {
 		if ( ! $sameRating)
 		{
 			if (
-				$dateDiff === 1 &&
+				$dateDiff === self::KITSU_GREATER &&
 				$kitsuItem['data']['rating'] !== 0 &&
 				$kitsuItem['data']['ratingTwenty'] !== 0
 			)
@@ -602,7 +601,7 @@ final class SyncLists extends BaseCommand {
 				$update['data']['ratingTwenty'] = $kitsuItem['data']['ratingTwenty'];
 				$return['updateType'][] = API::ANILIST;
 			}
-			else if($dateDiff === -1 && $anilistItem['data']['rating'] !== 0)
+			else if($dateDiff === self::ANILIST_GREATER && $anilistItem['data']['rating'] !== 0)
 			{
 				$update['data']['ratingTwenty'] = $anilistItem['data']['rating'] * 2;
 				$return['updateType'][] = API::KITSU;
@@ -612,7 +611,7 @@ final class SyncLists extends BaseCommand {
 		// If notes are set, use kitsu, otherwise, set kitsu from anilist
 		if ( ! $sameNotes)
 		{
-			if ($kitsuItem['data']['notes'] !== '')
+			if ( ! empty($kitsuItem['data']['notes']))
 			{
 				$update['data']['notes'] = $kitsuItem['data']['notes'];
 				$return['updateType'][] = API::ANILIST;
@@ -627,12 +626,12 @@ final class SyncLists extends BaseCommand {
 		// Assume the larger reconsumeCount is correct
 		if ( ! $sameRewatchCount)
 		{
-			if ($diff['reconsumeCount'] === 1)
+			if ($diff['reconsumeCount'] === self::KITSU_GREATER)
 			{
 				$update['data']['reconsumeCount'] = $kitsuItem['data']['reconsumeCount'];
 				$return['updateType'][] = API::ANILIST;
 			}
-			else if ($diff['reconsumeCount'] === -1)
+			else if ($diff['reconsumeCount'] === self::ANILIST_GREATER)
 			{
 				$update['data']['reconsumeCount'] = $anilistItem['data']['reconsumeCount'];
 				$return['updateType'][] = API::KITSU;
@@ -659,11 +658,14 @@ final class SyncLists extends BaseCommand {
 		// to handle each combination of fields
 		if ($return['updateType'][0] === API::ANILIST)
 		{
+			// Anilist GraphQL expects a rating from 1-100
 			$prevData = [
 				'notes' => $kitsuItem['data']['notes'],
 				'private' => $kitsuItem['data']['private'],
 				'progress' => $kitsuItem['data']['progress'],
-				'rating' => $kitsuItem['data']['ratingTwenty'] * 5,
+				// Transformed Kitsu data returns a rating from 1-10
+				// Anilist expects a rating from 1-100
+				'rating' => $kitsuItem['data']['rating'] * 10,
 				'reconsumeCount' => $kitsuItem['data']['reconsumeCount'],
 				'reconsuming' => $kitsuItem['data']['reconsuming'],
 				'status' => $kitsuItem['data']['status'],
@@ -677,6 +679,8 @@ final class SyncLists extends BaseCommand {
 				'notes' => $anilistItem['data']['notes'],
 				'private' => $anilistItem['data']['private'],
 				'progress' => $anilistItem['data']['progress'] ?? 0,
+				// Anilist returns a rating between 1-100
+				// Kitsu expects a rating from 1-20
 				'rating' => (((int)$anilistItem['data']['rating']) > 0)
 					? $anilistItem['data']['rating'] / 5
 					: 0,
@@ -824,30 +828,4 @@ final class SyncLists extends BaseCommand {
 			}
 		}
 	}
-
-	// ------------------------------------------------------------------------
-	// Other Helpers
-	// ------------------------------------------------------------------------
-
-	/**
-	 * Filter Kitsu mappings for the specified type
-	 *
-	 * @param array $includes
-	 * @param string $type
-	 * @return array
-	 */
-	private function filterMappings(array $includes, string $type = ListType::ANIME): array
-	{
-		$output = [];
-
-		foreach($includes as $id => $mapping)
-		{
-			if ($mapping['externalSite'] === "myanimelist/{$type}")
-			{
-				$output[$id] = $mapping;
-			}
-		}
-
-		return $output;
-	}
 }

src/AnimeClient/Enum/APISource.php

@@ -1,27 +0,0 @@
-<?php declare(strict_types=1);
-/**
- * Hummingbird Anime List Client
- *
- * An API client for Kitsu to manage anime and manga watch lists
- *
- * PHP version 7.4
- *
- * @package     HummingbirdAnimeClient
- * @author      Timothy J. Warren <tim@timshomepage.net>
- * @copyright   2015 - 2020  Timothy J. Warren
- * @license     http://www.opensource.org/licenses/mit-license.html  MIT License
- * @version     5.1
- * @link        https://git.timshomepage.net/timw4mail/HummingBirdAnimeClient
- */
-
-namespace Aviat\AnimeClient\Enum;
-
-use Aviat\Ion\Enum as BaseEnum;
-
-/**
- * Types of lists
- */
-final class APISource extends BaseEnum {
-	public const KITSU = 'kitsu';
-	public const ANILIST = 'anilist';
-}