timw4mail/HummingBirdAnimeClient
1
1
Fork 1
Code Issues 5 Pull Requests Projects 1 Releases Wiki Activity

Version 5.1 - All the GraphQL #32

Closed
timw4mail wants to merge 1160 commits from develop into master
pull from: develop
merge into: timw4mail:master
Conversation 0 Commits 1160 Files Changed 547 +503 -112
8 changed files with 503 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 08a882bbb6 - Show all commits

95
src/API/Kitsu/Model.php
View File

@ -555,6 +555,73 @@ class Model {
return $cacheItem->get();
}
/**
* Get the number of manga list items
*
* @param string $status - Optional status to filter by
* @return int
*/
public function getMangaListCount(string $status = '') : int
{
$options = [
'query' => [
'filter' => [
'user_id' => $this->getUserIdByUsername(),
'media_type' => 'Manga'
],
'page' => [
'limit' => 1
],
'sort' => '-updated_at'
]
];
if ( ! empty($status))
{
$options['query']['filter']['status'] = $status;
}
$response = $this->getRequest('library-entries', $options);
return $response['meta']['count'];
}
/**
* Get the full manga list
*
* @param array $options
* @return array
*/
public function getFullMangaList(array $options = [
'include' => 'manga.mappings'
]): array
{
$status = $options['filter']['status'] ?? '';
$count = $this->getMangaListCount($status);
$size = 100;
$pages = ceil($count / $size);
$requester = new ParallelAPIRequest();
// Set up requests
for ($i = 0; $i < $pages; $i++)
{
$offset = $i * $size;
$requester->addRequest($this->getPagedMangaList($size, $offset, $options));
}
$responses = $requester->makeRequests();
$output = [];
foreach($responses as $response)
{
$data = Json::decode($response->getBody());
$output = array_merge_recursive($output, $data);
}
return $output;
}
/**
* Get all Manga lists
*
@ -573,6 +640,34 @@ class Model {
return $output;
}
/**
* Get the full manga list in paginated form
*
* @param int $limit
* @param int $offset
* @param array $options
* @return Request
*/
public function getPagedMangaList(int $limit = 100, int $offset = 0, array $options = [
'include' => 'manga.mappings'
]): Request
{
$defaultOptions = [
'filter' => [
'user_id' => $this->getUserIdByUsername($this->getUsername()),
'media_type' => 'Manga'
],
'page' => [
'offset' => $offset,
'limit' => $limit
],
'sort' => '-updated_at'
];
$options = array_merge($defaultOptions, $options);
return $this->setUpRequest('GET', 'library-entries', ['query' => $options]);
}
/**
* Get the mal id for the manga represented by the kitsu id
* to enable updating MyAnimeList

34
src/API/MAL/ListItem.php
View File

@ -30,7 +30,14 @@ class ListItem {
use ContainerAware;
use MALTrait;
public function create(array $data): Request
/**
* Create a list item
*
* @param array $data
* @param string $type
* @return Request
*/
public function create(array $data, string $type = 'anime'): Request
{
$id = $data['id'];
$createData = [
@ -42,17 +49,24 @@ class ListItem {
$config = $this->container->get('config');
return $this->requestBuilder->newRequest('POST', "animelist/add/{$id}.xml")
return $this->requestBuilder->newRequest('POST', "{$type}list/add/{$id}.xml")
->setFormFields($createData)
->setBasicAuth($config->get(['mal','username']), $config->get(['mal', 'password']))
->getFullRequest();
}
public function delete(string $id): Request
/**
* Delete a list item
*
* @param string $id
* @param string $type
* @return Request
*/
public function delete(string $id, string $type = 'anime'): Request
{
$config = $this->container->get('config');
return $this->requestBuilder->newRequest('DELETE', "animelist/delete/{$id}.xml")
return $this->requestBuilder->newRequest('DELETE', "{$type}list/delete/{$id}.xml")
->setFormFields([
'id' => $id
])
@ -67,7 +81,15 @@ class ListItem {
return [];
}
public function update(string $id, array $data): Request
/**
* Update a list item
*
* @param string $id
* @param array $data
* @param string $type
* @return Request
*/
public function update(string $id, array $data, string $type = 'anime'): Request
{
$config = $this->container->get('config');
@ -76,7 +98,7 @@ class ListItem {
->addField('id', $id)
->addField('data', $xml);
return $this->requestBuilder->newRequest('POST', "animelist/update/{$id}.xml")
return $this->requestBuilder->newRequest('POST', "{$type}list/update/{$id}.xml")
->setFormFields([
'id' => $id,
'data' => $xml

69
src/API/MAL/Model.php
View File

@ -17,10 +17,9 @@
namespace Aviat\AnimeClient\API\MAL;
use Amp\Artax\Request;
use Aviat\AnimeClient\API\MAL\ListItem;
use Aviat\AnimeClient\API\MAL\Transformer\AnimeListTransformer;
use Aviat\AnimeClient\API\MAL\{ListItem, Transformer\AnimeListTransformer};
use Aviat\AnimeClient\API\XML;
use Aviat\AnimeClient\API\Mapping\AnimeWatchingStatus;
use Aviat\AnimeClient\API\Mapping\{AnimeWatchingStatus, MangaReadingStatus};
use Aviat\Ion\Di\ContainerAware;
/**
@ -51,12 +50,21 @@ class Model {
$this->listItem = $listItem;
}
public function createFullListItem(array $data): Request
/**
* Create a list item on MAL
*
* @param array $data
* @param string $type "anime" or "manga"
* @return Request
*/
public function createFullListItem(array $data, string $type = 'anime'): Request
{
return $this->listItem->create($data);
return $this->listItem->create($data, $type);
}
public function createListItem(array $data): Request
public function createListItem(array $data, string $type = 'anime'): Request
{
if ($type === 'anime')
{
$createData = [
'id' => $data['id'],
@ -64,25 +72,30 @@ class Model {
'status' => AnimeWatchingStatus::KITSU_TO_MAL[$data['status']]
]
];
}
elseif ($type === 'manga')
{
$createData = [
'id' => $data['id'],
'data' => [
'status' => MangaReadingStatus::KITSU_TO_MAL[$data['status']]
]
];
}
return $this->listItem->create($createData);
}
public function getFullList(): array
public function getMangaList(): array
{
$config = $this->container->get('config');
$userName = $config->get(['mal', 'username']);
$list = $this->getRequest('https://myanimelist.net/malappinfo.php', [
'headers' => [
'Accept' => 'text/xml'
],
'query' => [
'u' => $userName,
'status' => 'all'
]
]);
return $this->getList('manga');
}
return $list['myanimelist']['anime'];
public function getAnimeList(): array
{
return $this->getList('anime');
}
public function getListItem(string $listId): array
@ -100,4 +113,22 @@ class Model {
{
return $this->listItem->delete($id);
}
private function getList(string $type): array
{
$config = $this->container->get('config');
$userName = $config->get(['mal', 'username']);
$list = $this->getRequest('https://myanimelist.net/malappinfo.php', [
'headers' => [
'Accept' => 'text/xml'
],
'query' => [
'u' => $userName,
'status' => 'all',
'type' => $type
]
]);
return $list['myanimelist'][$type];
}
}

16
src/API/MAL/Transformer/AnimeListTransformer.php
View File

@ -24,26 +24,14 @@ use Aviat\Ion\Transformer\AbstractTransformer;
*/
class AnimeListTransformer extends AbstractTransformer {
/**
* Transform MAL episode data to Kitsu episode data
* Identity transformation
*
* @param array $item
* @return array
*/
public function transform($item)
{
$rewatching = (array_key_exists('rewatching', $item) && $item['rewatching']);
return [
'id' => $item['mal_id'],
'data' => [
'status' => AnimeWatchingStatus::KITSU_TO_MAL[$item['watching_status']],
'rating' => $item['user_rating'],
'rewatch_value' => (int) $rewatching,
'times_rewatched' => $item['rewatched'],
'comments' => $item['notes'],
'episode' => $item['episodes_watched']
]
];
return $item;
}
/**

85
src/API/MAL/Transformer/MangaListTransformer.php Normal file
View File

@ -0,0 +1,85 @@
<?php declare(strict_types=1);
/**
* Hummingbird Anime List Client
*
* An API client for Kitsu and MyAnimeList to manage anime and manga watch lists
*
* PHP version 7
*
* @package HummingbirdAnimeClient
* @author Timothy J. Warren <tim@timshomepage.net>
* @copyright 2015 - 2017 Timothy J. Warren
* @license http://www.opensource.org/licenses/mit-license.html MIT License
* @version 4.0
* @link https://git.timshomepage.net/timw4mail/HummingBirdAnimeClient
*/
namespace Aviat\AnimeClient\API\MAL\Transformer;
use Aviat\AnimeClient\API\Mapping\MangaReadingStatus;
use Aviat\Ion\Transformer\AbstractTransformer;
/**
* Transformer for updating MAL List
*/
class MangaListTransformer extends AbstractTransformer {
/**
* Identity transformation
*
* @param array $item
* @return array
*/
public function transform($item)
{
return $item;
}
/**
* Transform Kitsu data to MAL data
*
* @param array $item
* @return array
*/
public function untransform(array $item): array
{
$map = [
'id' => $item['mal_id'],
'data' => [
'chapter' => $item['data']['progress']
]
];
$data =& $item['data'];
foreach($item['data'] as $key => $value)
{
switch($key)
{
case 'notes':
$map['data']['comments'] = $value;
break;
case 'rating':
$map['data']['score'] = $value * 2;
break;
case 'reconsuming':
$map['data']['enable_rereading'] = (bool) $value;
break;
case 'reconsumeCount':
$map['data']['times_reread'] = $value;
break;
case 'status':
$map['data']['status'] = MangaReadingStatus::KITSU_TO_MAL[$value];
break;
default:
break;
}
}
return $map;
}
}

21
src/API/Mapping/MangaReadingStatus.php
View File

@ -29,7 +29,7 @@ use Aviat\Ion\Enum;
* and url route segments
*/
class MangaReadingStatus extends Enum {
const MAL_TO_KITSU = [
const KITSU_TO_MAL = [
Kitsu::READING => MAL::READING,
Kitsu::PLAN_TO_READ => MAL::PLAN_TO_READ,
Kitsu::COMPLETED => MAL::COMPLETED,
@ -37,12 +37,17 @@ class MangaReadingStatus extends Enum {
Kitsu::DROPPED => MAL::DROPPED
];
const KITSU_TO_MAL = [
const MAL_TO_KITSU = [
'1' => Kitsu::READING,
'2' => Kitsu::COMPLETED,
'3' => Kitsu::ON_HOLD,
'4' => Kitsu::DROPPED,
'6' => Kitsu::PLAN_TO_READ,
MAL::READING => Kitsu::READING,
MAL::PLAN_TO_READ => Kitsu::PLAN_TO_READ,
MAL::COMPLETED => Kitsu::COMPLETED,
MAL::ON_HOLD => Kitsu::ON_HOLD,
MAL::DROPPED => Kitsu::DROPPED
MAL::DROPPED => Kitsu::DROPPED,
MAL::PLAN_TO_READ => Kitsu::PLAN_TO_READ,
];
const KITSU_TO_TITLE = [
@ -50,7 +55,7 @@ class MangaReadingStatus extends Enum {
Kitsu::PLAN_TO_READ => Title::PLAN_TO_READ,
Kitsu::COMPLETED => Title::COMPLETED,
Kitsu::ON_HOLD => Title::ON_HOLD,
Kitsu::DROPPED => Title::DROPPED
Kitsu::DROPPED => Title::DROPPED,
];
const ROUTE_TO_KITSU = [
@ -58,7 +63,7 @@ class MangaReadingStatus extends Enum {
Route::READING => Kitsu::READING,
Route::COMPLETED => Kitsu::COMPLETED,
Route::DROPPED => Kitsu::DROPPED,
Route::ON_HOLD => Kitsu::ON_HOLD
Route::ON_HOLD => Kitsu::ON_HOLD,
];
const ROUTE_TO_TITLE = [
@ -67,7 +72,7 @@ class MangaReadingStatus extends Enum {
Route::READING => Title::READING,
Route::COMPLETED => Title::COMPLETED,
Route::DROPPED => Title::DROPPED,
Route::ON_HOLD => Title::ON_HOLD
Route::ON_HOLD => Title::ON_HOLD,
];
const TITLE_TO_KITSU = [
@ -75,6 +80,6 @@ class MangaReadingStatus extends Enum {
Title::READING => Kitsu::READING,
Title::COMPLETED => Kitsu::COMPLETED,
Title::DROPPED => Kitsu::DROPPED,
Title::ON_HOLD => Kitsu::ON_HOLD
Title::ON_HOLD => Kitsu::ON_HOLD,
];
}

268
src/Command/SyncKitsuWithMal.php
View File

@ -19,8 +19,16 @@ namespace Aviat\AnimeClient\Command;
use function Amp\{all, wait};
use Amp\Artax\Client;
use Aviat\AnimeClient\API\{JsonAPI, Mapping\AnimeWatchingStatus};
use Aviat\AnimeClient\API\MAL\Transformer\AnimeListTransformer as ALT;
use Aviat\AnimeClient\API\{
JsonAPI,
ParallelAPIRequest,
Mapping\AnimeWatchingStatus,
Mapping\MangaReadingStatus
};
use Aviat\AnimeClient\API\MAL\Transformer\{
AnimeListTransformer as ALT,
MangaListTransformer as MLT
};
use Aviat\Ion\Json;
/**
@ -55,71 +63,71 @@ class SyncKitsuWithMal extends BaseCommand {
$this->kitsuModel = $this->container->get('kitsu-model');
$this->malModel = $this->container->get('mal-model');
$malCount = count($this->getMALAnimeList());
$kitsuCount = $this->getKitsuAnimeListPageCount();
$this->syncAnime();
$this->syncManga();
}
$this->echoBox("Number of MAL list items: {$malCount}");
$this->echoBox("Number of Kitsu list items: {$kitsuCount}");
public function syncAnime()
{
$malCount = count($this->malModel->getAnimeList());
$kitsuCount = $this->kitsuModel->getAnimeListCount();
$this->echoBox("Number of MAL anime list items: {$malCount}");
$this->echoBox("Number of Kitsu anime list items: {$kitsuCount}");
$data = $this->diffAnimeLists();
$this->echoBox("Number of items that need to be added to MAL: " . count($data['addToMAL']));
$this->echoBox("Number of anime items that need to be added to MAL: " . count($data['addToMAL']));
if ( ! empty($data['addToMAL']))
{
$this->echoBox("Adding missing list items to MAL");
$this->echoBox("Adding missing anime list items to MAL");
$this->createMALAnimeListItems($data['addToMAL']);
}
$this->echoBox('Number of items that need to be added to Kitsu: ' . count($data['addToKitsu']));
$this->echoBox('Number of anime items that need to be added to Kitsu: ' . count($data['addToKitsu']));
if ( ! empty($data['addToKitsu']))
{
$this->echoBox("Adding missing list items to Kitsu");
$this->echoBox("Adding missing anime list items to Kitsu");
$this->createKitusAnimeListItems($data['addToKitsu']);
}
}
public function getKitsuAnimeList()
public function syncManga()
{
$count = $this->getKitsuAnimeListPageCount();
$size = 100;
$pages = ceil($count / $size);
$malCount = count($this->malModel->getMangaList());
$kitsuCount = $this->kitsuModel->getMangaListCount();
$requests = [];
$this->echoBox("Number of MAL manga list items: {$malCount}");
$this->echoBox("Number of Kitsu manga list items: {$kitsuCount}");
// Set up requests
for ($i = 0; $i < $pages; $i++)
$data = $this->diffMangaLists();
$this->echoBox("Number of manga items that need to be added to MAL: " . count($data['addToMAL']));
if ( ! empty($data['addToMAL']))
{
$offset = $i * $size;
$requests[] = $this->kitsuModel->getPagedAnimeList($size, $offset);
$this->echoBox("Adding missing manga list items to MAL");
$this->createMALMangaListItems($data['addToMAL']);
}
$promiseArray = (new Client())->requestMulti($requests);
$this->echoBox('Number of manga items that need to be added to Kitsu: ' . count($data['addToKitsu']));
$responses = wait(all($promiseArray));
$output = [];
foreach($responses as $response)
if ( ! empty($data['addToKitsu']))
{
$data = Json::decode($response->getBody());
$output = array_merge_recursive($output, $data);
$this->echoBox("Adding missing manga list items to Kitsu");
$this->createKitsuMangaListItems($data['addToKitsu']);
}
}
return $output;
}
public function getMALAnimeList()
{
return $this->malModel->getFullList();
}
public function filterMappings(array $includes): array
public function filterMappings(array $includes, string $type = 'anime'): array
{
$output = [];
foreach($includes as $id => $mapping)
{
if ($mapping['externalSite'] === 'myanimelist/anime')
if ($mapping['externalSite'] === "myanimelist/{$type}")
{
$output[$id] = $mapping;
}
@ -130,7 +138,7 @@ class SyncKitsuWithMal extends BaseCommand {
public function formatMALAnimeList()
{
$orig = $this->getMALAnimeList();
$orig = $this->malModel->getAnimeList();
$output = [];
foreach($orig as $item)
@ -156,6 +164,35 @@ class SyncKitsuWithMal extends BaseCommand {
return $output;
}
public function formatMALMangaList()
{
$orig = $this->malModel->getMangaList();
$output = [];
foreach($orig as $item)
{
$output[$item['series_mangadb_id']] = [
'id' => $item['series_mangadb_id'],
'data' => [
'my_status' => $item['my_status'],
'status' => MangaReadingStatus::MAL_TO_KITSU[$item['my_status']],
'progress' => $item['my_read_chapters'],
'reconsuming' => (bool) $item['my_rereadingg'],
/* 'reconsumeCount' => array_key_exists('times_rewatched', $item)
? $item['times_rewatched']
: 0, */
// 'notes' => ,
'rating' => $item['my_score'] / 2,
'updatedAt' => (new \DateTime())
->setTimestamp((int)$item['my_last_updated'])
->format(\DateTime::W3C),
]
];
}
return $output;
}
public function filterKitsuAnimeList()
{
$data = $this->kitsuModel->getFullAnimeList();
@ -194,9 +231,84 @@ class SyncKitsuWithMal extends BaseCommand {
return $output;
}
public function getKitsuAnimeListPageCount()
public function filterKitsuMangaList()
{
return $this->kitsuModel->getAnimeListCount();
$data = $this->kitsuModel->getFullMangaList();
$includes = JsonAPI::organizeIncludes($data['included']);
$includes['mappings'] = $this->filterMappings($includes['mappings'], 'manga');
$output = [];
foreach($data['data'] as $listItem)
{
$mangaId = $listItem['relationships']['manga']['data']['id'];
$potentialMappings = $includes['manga'][$mangaId]['relationships']['mappings'];
$malId = NULL;
foreach ($potentialMappings as $mappingId)
{
if (array_key_exists($mappingId, $includes['mappings']))
{
$malId = $includes['mappings'][$mappingId]['externalId'];
}
}
// Skip to the next item if there isn't a MAL ID
if (is_null($malId))
{
continue;
}
$output[$listItem['id']] = [
'id' => $listItem['id'],
'malId' => $malId,
'data' => $listItem['attributes'],
];
}
return $output;
}
public function diffMangaLists()
{
$kitsuList = $this->filterKitsuMangaList();
$malList = $this->formatMALMangaList();
$itemsToAddToMAL = [];
$itemsToAddToKitsu = [];
$malIds = array_column($malList, 'id');
$kitsuMalIds = array_column($kitsuList, 'malId');
$missingMalIds = array_diff($malIds, $kitsuMalIds);
foreach($missingMalIds as $mid)
{
$itemsToAddToKitsu[] = array_merge($malList[$mid]['data'], [
'id' => $this->kitsuModel->getKitsuIdFromMALId($mid, 'manga'),
'type' => 'manga'
]);
}
foreach($kitsuList as $kitsuItem)
{
if (in_array($kitsuItem['malId'], $malIds))
{
// Eventually, compare the list entries, and determine which
// needs to be updated
continue;
}
// Looks like this item only exists on Kitsu
$itemsToAddToMAL[] = [
'mal_id' => $kitsuItem['malId'],
'data' => $kitsuItem['data']
];
}
return [
'addToMAL' => $itemsToAddToMAL,
'addToKitsu' => $itemsToAddToKitsu
];
}
public function diffAnimeLists()
@ -256,29 +368,79 @@ class SyncKitsuWithMal extends BaseCommand {
];
}
public function createKitusAnimeListItems($itemsToAdd)
public function createKitsuMangaListItems($itemsToAdd)
{
$requests = [];
$requester = new ParallelAPIRequest();
foreach($itemsToAdd as $item)
{
$requests[] = $this->kitsuModel->createListItem($item);
$requester->addRequest($this->kitsuModel->createListItem($item));
}
$promiseArray = (new Client())->requestMulti($requests);
$responses = wait(all($promiseArray));
$responses = $requester->makeRequests();
foreach($responses as $key => $response)
{
$id = $itemsToAdd[$key]['id'];
if ($response->getStatus() === 201)
{
$this->echoBox("Successfully create list item with id: {$id}");
$this->echoBox("Successfully created Kitsu manga list item with id: {$id}");
}
else
{
echo $response->getBody();
$this->echoBox("Failed to create list item with id: {$id}");
$this->echoBox("Failed to create Kitsu manga list item with id: {$id}");
}
}
}
public function createMALMangaListItems($itemsToAdd)
{
$transformer = new MLT();
$requester = new ParallelAPIRequest();
foreach($itemsToAdd as $item)
{
$data = $transformer->untransform($item);
$requester->addRequest($this->malModel->createFullListItem($data, 'manga'));
}
$responses = $requester->makeRequests();
foreach($responses as $key => $response)
{
$id = $itemsToAdd[$key]['mal_id'];
if ($response->getBody() === 'Created')
{
$this->echoBox("Successfully created MAL manga list item with id: {$id}");
}
else
{
$this->echoBox("Failed to create MAL manga list item with id: {$id}");
}
}
}
public function createKitusAnimeListItems($itemsToAdd)
{
$requester = new ParallelAPIRequest();
foreach($itemsToAdd as $item)
{
$requester->addRequest($this->kitsuModel->createListItem($item));
}
$responses = $requester->makeRequests();
foreach($responses as $key => $response)
{
$id = $itemsToAdd[$key]['id'];
if ($response->getStatus() === 201)
{
$this->echoBox("Successfully created Kitsu anime list item with id: {$id}");
}
else
{
echo $response->getBody();
$this->echoBox("Failed to create Kitsu anime list item with id: {$id}");
}
}
}
@ -286,28 +448,26 @@ class SyncKitsuWithMal extends BaseCommand {
public function createMALAnimeListItems($itemsToAdd)
{
$transformer = new ALT();
$requests = [];
$requester = new ParallelAPIRequest();
foreach($itemsToAdd as $item)
{
$data = $transformer->untransform($item);
$requests[] = $this->malModel->createFullListItem($data);
$requester->addRequest($this->malModel->createFullListItem($data));
}
$promiseArray = (new Client())->requestMulti($requests);
$responses = wait(all($promiseArray));
$responses = $requester->makeRequests();
foreach($responses as $key => $response)
{
$id = $itemsToAdd[$key]['mal_id'];
if ($response->getBody() === 'Created')
{
$this->echoBox("Successfully create list item with id: {$id}");
$this->echoBox("Successfully created MAL anime list item with id: {$id}");
}
else
{
$this->echoBox("Failed to create list item with id: {$id}");
$this->echoBox("Failed to create MAL anime list item with id: {$id}");
}
}
}

15
src/Model/Anime.php
View File

@ -59,7 +59,7 @@ class Anime extends API {
* @param string $status
* @return array
*/
public function getList($status)
public function getList($status): array
{
$data = $this->kitsuModel->getAnimeList($status);
$this->sortByName($data, 'anime');
@ -72,7 +72,12 @@ class Anime extends API {
return $output;
}
public function getAllLists()
/**
* Get data for the 'all' anime page
*
* @return array
*/
public function getAllLists(): array
{
$data = $this->kitsuModel->getFullOrganizedAnimeList();
@ -90,7 +95,7 @@ class Anime extends API {
* @param string $slug
* @return array
*/
public function getAnime($slug)
public function getAnime(string $slug): array
{
return $this->kitsuModel->getAnime($slug);
}
@ -101,7 +106,7 @@ class Anime extends API {
* @param string $animeId
* @return array
*/
public function getAnimeById($animeId)
public function getAnimeById(string $animeId): array
{
return $this->kitsuModel->getAnimeById($animeId);
}
@ -112,7 +117,7 @@ class Anime extends API {
* @param string $name
* @return array
*/
public function search($name)
public function search(string $name): array
{
return $this->kitsuModel->search('anime', $name);
}