API client for Kitsu.io, with optional Anime collection, and optional Anilist syncing.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

SyncKitsuWithMal.php 13KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538
  1. <?php declare(strict_types=1);
  2. /**
  3. * Hummingbird Anime List Client
  4. *
  5. * An API client for Kitsu and MyAnimeList to manage anime and manga watch lists
  6. *
  7. * PHP version 7
  8. *
  9. * @package HummingbirdAnimeClient
  10. * @author Timothy J. Warren <tim@timshomepage.net>
  11. * @copyright 2015 - 2018 Timothy J. Warren
  12. * @license http://www.opensource.org/licenses/mit-license.html MIT License
  13. * @version 4.0
  14. * @link https://git.timshomepage.net/timw4mail/HummingBirdAnimeClient
  15. */
  16. namespace Aviat\AnimeClient\Command;
  17. use Aviat\AnimeClient\API\{
  18. FailedResponseException,
  19. JsonAPI,
  20. ParallelAPIRequest,
  21. Mapping\AnimeWatchingStatus,
  22. Mapping\MangaReadingStatus
  23. };
  24. use Aviat\AnimeClient\API\MAL\Transformer\{
  25. AnimeListTransformer as ALT
  26. };
  27. use Aviat\Ion\Json;
  28. /**
  29. * Clears the API Cache
  30. */
  31. class SyncKitsuWithMal extends BaseCommand {
  32. /**
  33. * Model for making requests to Kitsu API
  34. * @var \Aviat\AnimeClient\API\Kitsu\Model
  35. */
  36. protected $kitsuModel;
  37. /**
  38. * Model for making requests to MAL API
  39. * @var \Aviat\AnimeClient\API\MAL\Model
  40. */
  41. protected $malModel;
  42. /**
  43. * Run the Kitsu <=> MAL sync script
  44. *
  45. * @param array $args
  46. * @param array $options
  47. * @return void
  48. * @throws \ConsoleKit\ConsoleException
  49. */
  50. public function execute(array $args, array $options = [])
  51. {
  52. $this->setContainer($this->setupContainer());
  53. $this->setCache($this->container->get('cache'));
  54. $this->kitsuModel = $this->container->get('kitsu-model');
  55. $this->malModel = $this->container->get('mal-model');
  56. $this->sync('anime');
  57. $this->sync('manga');
  58. }
  59. public function sync(string $type)
  60. {
  61. $uType = ucfirst($type);
  62. // Do a little check to make sure you don't have immediate issues
  63. // if you have 0 or 1 items in a list on MAL.
  64. $malList = $this->malModel->getList($type);
  65. $malCount = 0;
  66. if ( ! empty($malList))
  67. {
  68. $malCount = count(array_key_exists(0, $malList)
  69. ? $malList
  70. : [$malList]
  71. );
  72. }
  73. try
  74. {
  75. $kitsuCount = $this->kitsuModel->{"get{$uType}ListCount"}();
  76. }
  77. catch (FailedResponseException $e)
  78. {
  79. dump($e);
  80. }
  81. $this->echoBox("Number of MAL {$type} list items: {$malCount}");
  82. $this->echoBox("Number of Kitsu {$type} list items: {$kitsuCount}");
  83. $data = $this->diffLists($type);
  84. if ( ! empty($data['addToMAL']))
  85. {
  86. $count = count($data['addToMAL']);
  87. $this->echoBox("Adding {$count} missing {$type} list items to MAL");
  88. $this->updateMALListItems($data['addToMAL'], 'create', $type);
  89. }
  90. if ( ! empty($data['updateMAL']))
  91. {
  92. $count = count($data['updateMAL']);
  93. $this->echoBox("Updating {$count} outdated MAL {$type} list items");
  94. $this->updateMALListItems($data['updateMAL'], 'update', $type);
  95. }
  96. if ( ! empty($data['addToKitsu']))
  97. {
  98. $count = count($data['addToKitsu']);
  99. $this->echoBox("Adding {$count} missing {$type} list items to Kitsu");
  100. $this->updateKitsuListItems($data['addToKitsu'], 'create', $type);
  101. }
  102. if ( ! empty($data['updateKitsu']))
  103. {
  104. $count = count($data['updateKitsu']);
  105. $this->echoBox("Updating {$count} outdated Kitsu {$type} list items");
  106. $this->updateKitsuListItems($data['updateKitsu'], 'update', $type);
  107. }
  108. }
  109. public function filterMappings(array $includes, string $type = 'anime'): array
  110. {
  111. $output = [];
  112. foreach($includes as $id => $mapping)
  113. {
  114. if ($mapping['externalSite'] === "myanimelist/{$type}")
  115. {
  116. $output[$id] = $mapping;
  117. }
  118. }
  119. return $output;
  120. }
  121. public function formatMALList(string $type): array
  122. {
  123. if ($type === 'anime')
  124. {
  125. return $this->formatMALAnimeList();
  126. }
  127. if ($type === 'manga')
  128. {
  129. return $this->formatMALMangaList();
  130. }
  131. }
  132. public function formatMALAnimeList()
  133. {
  134. $orig = $this->malModel->getList('anime');
  135. $output = [];
  136. // Bail early on empty list
  137. if (empty($orig))
  138. {
  139. return [];
  140. }
  141. // Due to xml parsing differences,
  142. // 1 item has no wrapping array.
  143. // In this case, just re-create the
  144. // wrapper array
  145. if ( ! array_key_exists(0, $orig))
  146. {
  147. $orig = [$orig];
  148. }
  149. foreach($orig as $item)
  150. {
  151. $output[$item['series_animedb_id']] = [
  152. 'id' => $item['series_animedb_id'],
  153. 'data' => [
  154. 'status' => AnimeWatchingStatus::MAL_TO_KITSU[$item['my_status']],
  155. 'progress' => $item['my_watched_episodes'],
  156. 'reconsuming' => (bool) $item['my_rewatching'],
  157. 'rating' => $item['my_score'] / 2,
  158. 'updatedAt' => (new \DateTime())
  159. ->setTimestamp((int)$item['my_last_updated'])
  160. ->format(\DateTime::W3C),
  161. ]
  162. ];
  163. }
  164. return $output;
  165. }
  166. public function formatMALMangaList()
  167. {
  168. $orig = $this->malModel->getList('manga');
  169. $output = [];
  170. // Bail early on empty list
  171. if (empty($orig))
  172. {
  173. return [];
  174. }
  175. // Due to xml parsing differences,
  176. // 1 item has no wrapping array.
  177. // In this case, just re-create the
  178. // wrapper array
  179. if ( ! array_key_exists(0, $orig))
  180. {
  181. $orig = [$orig];
  182. }
  183. foreach($orig as $item)
  184. {
  185. $output[$item['series_mangadb_id']] = [
  186. 'id' => $item['series_mangadb_id'],
  187. 'data' => [
  188. 'my_status' => $item['my_status'],
  189. 'status' => MangaReadingStatus::MAL_TO_KITSU[$item['my_status']],
  190. 'progress' => $item['my_read_chapters'],
  191. 'volumes' => $item['my_read_volumes'],
  192. 'reconsuming' => (bool) $item['my_rereadingg'],
  193. 'rating' => $item['my_score'] / 2,
  194. 'updatedAt' => (new \DateTime())
  195. ->setTimestamp((int)$item['my_last_updated'])
  196. ->format(\DateTime::W3C),
  197. ]
  198. ];
  199. }
  200. return $output;
  201. }
  202. public function formatKitsuList(string $type = 'anime'): array
  203. {
  204. $data = $this->kitsuModel->{'getFull' . ucfirst($type) . 'List'}();
  205. if (empty($data))
  206. {
  207. return [];
  208. }
  209. $includes = JsonAPI::organizeIncludes($data['included']);
  210. $includes['mappings'] = $this->filterMappings($includes['mappings'], $type);
  211. $output = [];
  212. foreach($data['data'] as $listItem)
  213. {
  214. $id = $listItem['relationships'][$type]['data']['id'];
  215. $potentialMappings = $includes[$type][$id]['relationships']['mappings'];
  216. $malId = NULL;
  217. foreach ($potentialMappings as $mappingId)
  218. {
  219. if (array_key_exists($mappingId, $includes['mappings']))
  220. {
  221. $malId = $includes['mappings'][$mappingId]['externalId'];
  222. }
  223. }
  224. // Skip to the next item if there isn't a MAL ID
  225. if (is_null($malId))
  226. {
  227. continue;
  228. }
  229. $output[$listItem['id']] = [
  230. 'id' => $listItem['id'],
  231. 'malId' => $malId,
  232. 'data' => $listItem['attributes'],
  233. ];
  234. }
  235. return $output;
  236. }
  237. public function diffLists(string $type = 'anime'): array
  238. {
  239. // Get libraryEntries with media.mappings from Kitsu
  240. // Organize mappings, and ignore entries without mappings
  241. $kitsuList = $this->formatKitsuList($type);
  242. // Get MAL list data
  243. $malList = $this->formatMALList($type);
  244. $itemsToAddToMAL = [];
  245. $itemsToAddToKitsu = [];
  246. $malUpdateItems = [];
  247. $kitsuUpdateItems = [];
  248. $malIds = array_column($malList, 'id');
  249. $kitsuMalIds = array_column($kitsuList, 'malId');
  250. $missingMalIds = array_diff($malIds, $kitsuMalIds);
  251. foreach($missingMalIds as $mid)
  252. {
  253. $itemsToAddToKitsu[] = array_merge($malList[$mid]['data'], [
  254. 'id' => $this->kitsuModel->getKitsuIdFromMALId($mid, $type),
  255. 'type' => $type
  256. ]);
  257. }
  258. foreach($kitsuList as $kitsuItem)
  259. {
  260. if (in_array($kitsuItem['malId'], $malIds))
  261. {
  262. $item = $this->compareListItems($kitsuItem, $malList[$kitsuItem['malId']]);
  263. if (is_null($item))
  264. {
  265. continue;
  266. }
  267. if (in_array('kitsu', $item['updateType']))
  268. {
  269. $kitsuUpdateItems[] = $item['data'];
  270. }
  271. if (in_array('mal', $item['updateType']))
  272. {
  273. $malUpdateItems[] = $item['data'];
  274. }
  275. continue;
  276. }
  277. // Looks like this item only exists on Kitsu
  278. $itemsToAddToMAL[] = [
  279. 'mal_id' => $kitsuItem['malId'],
  280. 'data' => $kitsuItem['data']
  281. ];
  282. }
  283. return [
  284. 'addToMAL' => $itemsToAddToMAL,
  285. 'updateMAL' => $malUpdateItems,
  286. 'addToKitsu' => $itemsToAddToKitsu,
  287. 'updateKitsu' => $kitsuUpdateItems
  288. ];
  289. }
  290. public function compareListItems(array $kitsuItem, array $malItem)
  291. {
  292. $compareKeys = ['status', 'progress', 'rating', 'reconsuming'];
  293. $diff = [];
  294. $dateDiff = (new \DateTime($kitsuItem['data']['updatedAt'])) <=> (new \DateTime($malItem['data']['updatedAt']));
  295. foreach($compareKeys as $key)
  296. {
  297. $diff[$key] = $kitsuItem['data'][$key] <=> $malItem['data'][$key];
  298. }
  299. // No difference? Bail out early
  300. $diffValues = array_values($diff);
  301. $diffValues = array_unique($diffValues);
  302. if (count($diffValues) === 1 && $diffValues[0] === 0)
  303. {
  304. return;
  305. }
  306. $update = [
  307. 'id' => $kitsuItem['id'],
  308. 'mal_id' => $kitsuItem['malId'],
  309. 'data' => []
  310. ];
  311. $return = [
  312. 'updateType' => []
  313. ];
  314. $sameStatus = $diff['status'] === 0;
  315. $sameProgress = $diff['progress'] === 0;
  316. $sameRating = $diff['rating'] === 0;
  317. // If status is the same, and progress count is different, use greater progress
  318. if ($sameStatus && ( ! $sameProgress))
  319. {
  320. if ($diff['progress'] === 1)
  321. {
  322. $update['data']['progress'] = $kitsuItem['data']['progress'];
  323. $return['updateType'][] = 'mal';
  324. }
  325. else if($diff['progress'] === -1)
  326. {
  327. $update['data']['progress'] = $malItem['data']['progress'];
  328. $return['updateType'][] = 'kitsu';
  329. }
  330. }
  331. // If status and progress are different, it's a bit more complicated...
  332. // But, at least for now, assume newer record is correct
  333. if ( ! ($sameStatus || $sameProgress))
  334. {
  335. if ($dateDiff === 1)
  336. {
  337. $update['data']['status'] = $kitsuItem['data']['status'];
  338. if ((int)$kitsuItem['data']['progress'] !== 0)
  339. {
  340. $update['data']['progress'] = $kitsuItem['data']['progress'];
  341. }
  342. $return['updateType'][] = 'mal';
  343. }
  344. else if($dateDiff === -1)
  345. {
  346. $update['data']['status'] = $malItem['data']['status'];
  347. if ((int)$malItem['data']['progress'] !== 0)
  348. {
  349. $update['data']['progress'] = $kitsuItem['data']['progress'];
  350. }
  351. $return['updateType'][] = 'kitsu';
  352. }
  353. }
  354. // If rating is different, use the rating from the item most recently updated
  355. if ( ! $sameRating)
  356. {
  357. if ($dateDiff === 1)
  358. {
  359. $update['data']['rating'] = $kitsuItem['data']['rating'];
  360. $return['updateType'][] = 'mal';
  361. }
  362. else if ($dateDiff === -1)
  363. {
  364. $update['data']['rating'] = $malItem['data']['rating'];
  365. $return['updateType'][] = 'kitsu';
  366. }
  367. }
  368. // If status is different, use the status of the more recently updated item
  369. if ( ! $sameStatus)
  370. {
  371. if ($dateDiff === 1)
  372. {
  373. $update['data']['status'] = $kitsuItem['data']['status'];
  374. $return['updateType'][] = 'mal';
  375. }
  376. else if ($dateDiff === -1)
  377. {
  378. $update['data']['status'] = $malItem['data']['status'];
  379. $return['updateType'][] = 'kitsu';
  380. }
  381. }
  382. $return['meta'] = [
  383. 'kitsu' => $kitsuItem['data'],
  384. 'mal' => $malItem['data'],
  385. 'dateDiff' => $dateDiff,
  386. 'diff' => $diff,
  387. ];
  388. $return['data'] = $update;
  389. $return['updateType'] = array_unique($return['updateType']);
  390. return $return;
  391. }
  392. public function updateKitsuListItems($itemsToUpdate, string $action = 'update', string $type = 'anime'): void
  393. {
  394. $requester = new ParallelAPIRequest();
  395. foreach($itemsToUpdate as $item)
  396. {
  397. if ($action === 'update')
  398. {
  399. $requester->addRequest($this->kitsuModel->updateListItem($item));
  400. }
  401. else if ($action === 'create')
  402. {
  403. $requester->addRequest($this->kitsuModel->createListItem($item));
  404. }
  405. }
  406. $responses = $requester->makeRequests();
  407. foreach($responses as $key => $response)
  408. {
  409. $responseData = Json::decode($response);
  410. $id = $itemsToUpdate[$key]['id'];
  411. if ( ! array_key_exists('errors', $responseData))
  412. {
  413. $verb = ($action === 'update') ? 'updated' : 'created';
  414. $this->echoBox("Successfully {$verb} Kitsu {$type} list item with id: {$id}");
  415. }
  416. else
  417. {
  418. dump($responseData);
  419. $verb = ($action === 'update') ? 'update' : 'create';
  420. $this->echoBox("Failed to {$verb} Kitsu {$type} list item with id: {$id}");
  421. }
  422. }
  423. }
  424. public function updateMALListItems($itemsToUpdate, string $action = 'update', string $type = 'anime'): void
  425. {
  426. $transformer = new ALT();
  427. $requester = new ParallelAPIRequest();
  428. foreach($itemsToUpdate as $item)
  429. {
  430. if ($action === 'update')
  431. {
  432. $requester->addRequest($this->malModel->updateListItem($item, $type));
  433. }
  434. else if ($action === 'create')
  435. {
  436. $data = $transformer->untransform($item);
  437. $requester->addRequest($this->malModel->createFullListItem($data, $type));
  438. }
  439. }
  440. $responses = $requester->makeRequests();
  441. foreach($responses as $key => $response)
  442. {
  443. $id = $itemsToUpdate[$key]['mal_id'];
  444. $goodResponse = (
  445. ($action === 'update' && $response === 'Updated') ||
  446. ($action === 'create' && $response === 'Created')
  447. );
  448. if ($goodResponse)
  449. {
  450. $verb = ($action === 'update') ? 'updated' : 'created';
  451. $this->echoBox("Successfully {$verb} MAL {$type} list item with id: {$id}");
  452. }
  453. else
  454. {
  455. $verb = ($action === 'update') ? 'update' : 'create';
  456. $this->echoBox("Failed to {$verb} MAL {$type} list item with id: {$id}");
  457. }
  458. }
  459. }
  460. }