29d41726b27ffec60b333c1e8493a84d13a50993
[stouyapi.git] / bin / import-game-data.php
1 #!/usr/bin/env php
2 <?php
3 /**
4  * Import games from a OUYA game data repository
5  *
6  * @link https://github.com/cweiske/ouya-game-data/
7  * @author Christian Weiske <cweiske@cweiske.de>
8  */
9 ini_set('xdebug.halt_level', E_WARNING|E_NOTICE|E_USER_WARNING|E_USER_NOTICE);
10 require_once __DIR__ . '/filters.php';
11 if (!isset($argv[1])) {
12     error('Pass the path to a "folders" file with game data json files folder names');
13 }
14 $foldersFile = $argv[1];
15 if (!is_file($foldersFile)) {
16     error('Given path is not a file: ' . $foldersFile);
17 }
18
19 $GLOBALS['packagelists']['cweiskepicks'] = [
20     'de.eiswuxe.blookid2',
21     'com.cosmos.babyloniantwins'
22 ];
23
24 $wwwDir = __DIR__ . '/../www/';
25
26 $baseDir   = dirname($foldersFile);
27 $gameFiles = [];
28 foreach (file($foldersFile) as $line) {
29     $line = trim($line);
30     if (strlen($line)) {
31         if (strpos($line, '..') !== false) {
32             error('Path attack in ' . $folder);
33         }
34         $folder = $baseDir . '/' . $line;
35         if (!is_dir($folder)) {
36             error('Folder does not exist: ' . $folder);
37         }
38         $gameFiles = array_merge($gameFiles, glob($folder . '/*.json'));
39     }
40 }
41
42 $games = [];
43 $count = 0;
44 $developers = [];
45 foreach ($gameFiles as $gameFile) {
46     $game = json_decode(file_get_contents($gameFile));
47     if ($game === null) {
48         error('JSON invalid at ' . $gameFile);
49     }
50     addMissingGameProperties($game);
51     $games[$game->packageName] = $game;
52
53     writeJson(
54         'api/v1/details-data/' . $game->packageName . '.json',
55         buildDetails($game)
56     );
57
58     if (!isset($developers[$game->developer->uuid])) {
59         $developers[$game->developer->uuid] = [
60             'info'     => $game->developer,
61             'products' => [],
62         ];
63     }
64
65     $products = $game->products ?? [];
66     foreach ($products as $product) {
67         writeJson(
68             'api/v1/developers/' . $game->developer->uuid
69             . '/products/' . $product->identifier . '.json',
70             buildDeveloperProductOnly($product, $game->developer)
71         );
72         $developers[$game->developer->uuid]['products'][] = $product;
73     }
74
75     /**/
76     writeJson(
77         'api/v1/games/' . $game->packageName . '/purchases',
78         buildPurchases($game)
79     );
80
81     writeJson(
82         'api/v1/apps/' . $game->packageName . '.json',
83         buildApps($game)
84     );
85     $latestRelease = $game->latestRelease;
86     writeJson(
87         'api/v1/apps/' . $latestRelease->uuid . '.json',
88         buildApps($game)
89     );
90
91     writeJson(
92         'api/v1/apps/' . $latestRelease->uuid . '-download.json',
93         buildAppDownload($game, $latestRelease)
94     );
95
96     if ($count++ > 20) {
97         //break;
98     }
99 }
100
101 calculateRank($games);
102
103 foreach ($developers as $developer) {
104     writeJson(
105         //index.htm does not need a rewrite rule
106         'api/v1/developers/' . $developer['info']->uuid
107         . '/products/index.htm',
108         buildDeveloperProducts($developer['products'], $developer['info'])
109     );
110     writeJson(
111         //index.htm does not need a rewrite rule
112         'api/v1/developers/' . $developer['info']->uuid
113         . '/current_gamer',
114         buildDeveloperCurrentGamer()
115     );
116 }
117
118 writeJson('api/v1/discover-data/discover.json', buildDiscover($games));
119 writeJson('api/v1/discover-data/home.json', buildDiscoverHome($games));
120
121 //make
122 writeJson(
123     'api/v1/discover-data/tutorials.json',
124     buildMakeCategory('Tutorials', filterByGenre($games, 'Tutorials'))
125 );
126
127 $searchLetters = 'abcdefghijklmnopqrstuvwxyz0123456789., ';
128 foreach (str_split($searchLetters) as $letter) {
129     $letterGames = filterBySearchWord($games, $letter);
130     writeJson(
131         'api/v1/search-data/' . $letter . '.json',
132         buildSearch($letterGames)
133     );
134 }
135
136
137 function buildDiscover(array $games)
138 {
139     $games = removeMakeGames($games);
140     $data = [
141         'title' => 'DISCOVER',
142         'rows'  => [],
143         'tiles' => [],
144     ];
145
146     addDiscoverRow(
147         $data, 'Last Updated',
148         filterLastUpdated($games, 10)
149     );
150     addDiscoverRow(
151         $data, 'Best rated',
152         filterBestRated($games, 10)
153     );
154     addDiscoverRow(
155         $data, "cweiske's picks",
156         filterByPackageNames($games, $GLOBALS['packagelists']['cweiskepicks'])
157     );
158
159     $players = [
160         //1 => '1 player',
161         2 => '2 players',
162         3 => '3 players',
163         4 => '4 players',
164     ];
165     addDiscoverRow($data, '# of players', $players);
166     foreach ($players as $num => $title) {
167         writeJson(
168             'api/v1/discover-data/' . categoryPath($title) . '.json',
169             buildDiscoverCategory($title, filterByPlayers($games, $num))
170         );
171     }
172
173     $ages = getAllAges($games);
174     natsort($ages);
175     addDiscoverRow($data, 'Content rating', $ages);
176     foreach ($ages as $num => $title) {
177         writeJson(
178             'api/v1/discover-data/' . categoryPath($title) . '.json',
179             buildDiscoverCategory($title, filterByAge($games, $title))
180         );
181     }
182
183     $genres = removeMakeGenres(getAllGenres($games));
184     sort($genres);
185     addChunkedDiscoverRows($data, $genres, 'Genres');
186
187     foreach ($genres as $genre) {
188         writeJson(
189             'api/v1/discover-data/' . categoryPath($genre) . '.json',
190             buildDiscoverCategory($genre, filterByGenre($games, $genre))
191         );
192     }
193
194     $abc = array_merge(range('A', 'Z'), ['Other']);
195     addChunkedDiscoverRows($data, $abc, 'Alphabetical');
196     foreach ($abc as $letter) {
197         writeJson(
198             'api/v1/discover-data/' . categoryPath($letter) . '.json',
199             buildDiscoverCategory($letter, filterByLetter($games, $letter))
200         );
201     }
202
203     return $data;
204 }
205
206 /**
207  * A genre category page
208  */
209 function buildDiscoverCategory($name, $games)
210 {
211     $data = [
212         'title' => $name,
213         'rows'  => [],
214         'tiles' => [],
215     ];
216     addDiscoverRow(
217         $data, 'Last Updated',
218         filterLastUpdated($games, 10)
219     );
220     addDiscoverRow(
221         $data, 'Best rated',
222         filterBestRated($games, 10)
223     );
224
225     $games = sortByTitle($games);
226     $chunks = array_chunk($games, 4);
227     foreach ($chunks as $chunkGames) {
228         addDiscoverRow($data, '', $chunkGames);
229     }
230
231     return $data;
232 }
233
234 function buildMakeCategory($name, $games)
235 {
236     $data = [
237         'title' => $name,
238         'rows'  => [],
239         'tiles' => [],
240     ];
241
242     $games = sortByTitle($games);
243     addDiscoverRow($data, '', $games);
244
245     return $data;
246 }
247
248 function buildDiscoverHome(array $games)
249 {
250     //we do not want anything here for now
251     $data = [
252         'title' => 'home',
253         'rows'  => [
254             [
255                 'title' => 'FEATURED',
256                 'showPrice' => false,
257                 'ranked'    => false,
258                 'tiles'     => [],
259             ]
260         ],
261         'tiles' => [],
262     ];
263     return $data;
264 }
265
266 /**
267  * Build api/v1/apps/$packageName
268  */
269 function buildApps($game)
270 {
271     $latestRelease = $game->latestRelease;
272
273     $product      = null;
274     $gamePromoted = getPromotedProduct($game);
275     if ($gamePromoted) {
276         $product = buildProduct($gamePromoted);
277     }
278
279     // http://cweiske.de/ouya-store-api-docs.htm#get-https-devs-ouya-tv-api-v1-apps-xxx
280     return [
281         'app' => [
282             'uuid'          => $latestRelease->uuid,
283             'title'         => $game->title,
284             'overview'      => $game->overview,
285             'description'   => $game->description,
286             'gamerNumbers'  => $game->players,
287             'genres'        => $game->genres,
288
289             'website'       => $game->website,
290             'contentRating' => $game->contentRating,
291             'premium'       => $game->premium,
292             'firstPublishedAt' => $game->firstPublishedAt,
293
294             'likeCount'     => $game->rating->likeCount,
295             'ratingAverage' => $game->rating->average,
296             'ratingCount'   => $game->rating->count,
297
298             'versionNumber' => $latestRelease->name,
299             'latestVersion' => $latestRelease->uuid,
300             'md5sum'        => $latestRelease->md5sum,
301             'apkFileSize'   => $latestRelease->size,
302             'publishedAt'   => $latestRelease->date,
303             'publicSize'    => $latestRelease->publicSize,
304             'nativeSize'    => $latestRelease->nativeSize,
305
306             'mainImageFullUrl' => $game->discover,
307             'videoUrl'         => getFirstVideoUrl($game->media),
308             'filepickerScreenshots' => getAllImageUrls($game->media),
309             'mobileAppIcon'    => null,
310
311             'developer'           => $game->developer->name,
312             'supportEmailAddress' => $game->developer->supportEmail,
313             'supportPhone'        => $game->developer->supportPhone,
314             'founder'             => $game->developer->founder,
315
316             'promotedProduct' => $product,
317         ],
318     ];
319 }
320
321 function buildAppDownload($game, $release)
322 {
323     return [
324         'app' => [
325             'fileSize'      => $release->size,
326             'version'       => $release->uuid,
327             'contentRating' => $game->contentRating,
328             'downloadLink'  => $release->url,
329         ]
330     ];
331 }
332
333 function buildProduct($product)
334 {
335     if ($product === null) {
336         return null;
337     }
338     return [
339         'type'          => 'entitlement',
340         'identifier'    => $product->identifier,
341         'name'          => $product->name,
342         'description'   => $product->description ?? '',
343         'localPrice'    => $product->localPrice,
344         'originalPrice' => $product->originalPrice,
345         'percentOff'    => 0,
346         'currency'      => $product->currency,
347     ];
348 }
349
350 /**
351  * Build /app/v1/details?app=org.example.game
352  */
353 function buildDetails($game)
354 {
355     $latestRelease = $game->latestRelease;
356
357     $mediaTiles = [];
358     if ($game->discover) {
359         $mediaTiles[] = [
360             'type' => 'image',
361             'urls' => [
362                 'thumbnail' => $game->discover,
363                 'full'      => $game->discover,
364             ],
365         ];
366     }
367     foreach ($game->media as $medium) {
368         if ($medium->type == 'image')  {
369             $mediaTiles[] = [
370                 'type' => 'image',
371                 'urls' => [
372                     'thumbnail' => $medium->thumb,
373                     'full'      => $medium->url,
374                 ],
375             ];
376         } else {
377             $mediaTiles[] = [
378                 'type' => 'video',
379                 'url'  => $medium->url,
380             ];
381         }
382     }
383
384     $buttons = [];
385     if (isset($game->links->unlocked)) {
386         $buttons[] = [
387             'text' => 'Show unlocked',
388             'url'  => 'ouya://launcher/details?app=' . $game->links->unlocked,
389             'bold' => true,
390         ];
391     }
392
393     $product      = null;
394     $gamePromoted = getPromotedProduct($game);
395     if ($gamePromoted) {
396         $product = buildProduct($gamePromoted);
397     }
398
399     // http://cweiske.de/ouya-store-api-docs.htm#get-https-devs-ouya-tv-api-v1-details
400     return [
401         'type'             => 'Game',
402         'title'            => $game->title,
403         'description'      => $game->description,
404         'gamerNumbers'     => $game->players,
405         'genres'           => $game->genres,
406
407         'suggestedAge'     => $game->contentRating,
408         'premium'          => $game->premium,
409         'inAppPurchases'   => $game->inAppPurchases,
410         'firstPublishedAt' => strtotime($game->firstPublishedAt),
411         'ccUrl'            => null,
412
413         'rating' => [
414             'count'   => $game->rating->count,
415             'average' => $game->rating->average,
416         ],
417
418         'apk' => [
419             'fileSize'    => $latestRelease->size,
420             'nativeSize'  => $latestRelease->nativeSize,
421             'publicSize'  => $latestRelease->publicSize,
422             'md5sum'      => $latestRelease->md5sum,
423             'filename'    => 'FIXME',
424             'errors'      => '',
425             'package'     => $game->packageName,
426             'versionCode' => $latestRelease->versionCode,
427             'state'       => 'complete',
428         ],
429
430         'version' => [
431             'number'      => $latestRelease->name,
432             'publishedAt' => strtotime($latestRelease->date),
433             'uuid'        => $latestRelease->uuid,
434         ],
435
436         'developer' => [
437             'name'    => $game->developer->name,
438             'founder' => $game->developer->founder,
439         ],
440
441         'metaData' => [
442             'key:rating.average',
443             'key:developer.name',
444             'key:suggestedAge',
445             number_format($latestRelease->size / 1024 / 1024, 2, '.', '') . ' MiB',
446         ],
447
448         'tileImage'     => $game->discover,
449         'mediaTiles'    => $mediaTiles,
450         'mobileAppIcon' => null,
451         'heroImage'     => [
452             'url' => null,
453         ],
454
455         'promotedProduct' => $product,
456         'buttons'         => $buttons,
457     ];
458 }
459
460 function buildDeveloperCurrentGamer()
461 {
462     return [
463         'gamer' => [
464             'uuid'     => '00702342-0000-1111-2222-c3e1500cafe2',
465             'username' => 'stouyapi',
466         ],
467     ];
468 }
469
470 /**
471  * For /api/v1/developers/xxx/products/?only=yyy
472  */
473 function buildDeveloperProductOnly($product, $developer)
474 {
475     return [
476         'developerName' => $developer->name,
477         'currency'      => $product->currency,
478         'products'      => [
479             buildProduct($product),
480         ],
481     ];
482 }
483
484 /**
485  * For /api/v1/developers/xxx/products/
486  */
487 function buildDeveloperProducts($products, $developer)
488 {
489     $jsonProducts = [];
490     foreach ($products as $product) {
491         $jsonProducts[] = buildProduct($product);
492     }
493     return [
494         'developerName' => $developer->name,
495         'currency'      => $products[0]->currency ?? 'EUR',
496         'products'      => $jsonProducts,
497     ];
498 }
499
500 function buildPurchases($game)
501 {
502     $purchasesData = [
503         'purchases' => [],
504     ];
505     $promotedProduct = getPromotedProduct($game);
506     if ($promotedProduct) {
507         $purchasesData['purchases'][] = [
508             'purchaseDate' => time() * 1000,
509             'generateDate' => time() * 1000,
510             'identifier'   => $promotedProduct->identifier,
511             'gamer'        => 'stouyapi',
512             'uuid'         => '00702342-0000-1111-2222-c3e1500cafe2',//gamer uuid
513             'priceInCents' => $promotedProduct->originalPrice * 100,
514             'localPrice'   => $promotedProduct->localPrice,
515             'currency'     => $promotedProduct->currency,
516         ];
517     }
518
519     $encryptedOnce  = dummyEncrypt($purchasesData);
520     $encryptedTwice = dummyEncrypt($encryptedOnce);
521     return $encryptedTwice;
522 }
523
524 function buildSearch($games)
525 {
526     $games = sortByTitle($games);
527     $results = [];
528     foreach ($games as $game) {
529         $results[] = [
530             'title' => $game->title,
531             'url'   => 'ouya://launcher/details?app=' . $game->packageName,
532             'contentRating' => $game->contentRating,
533         ];
534     }
535     return [
536         'count'   => count($results),
537         'results' => $results,
538     ];
539 }
540
541 function dummyEncrypt($data)
542 {
543     return [
544         'key'  => base64_encode('0123456789abcdef') . "\n",
545         'iv'   => 't3jir1LHpICunvhlM76edQ==' . "\n",//random bytes
546         'blob' => base64_encode(
547             json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)
548         ) . "\n",
549     ];
550 }
551
552 function addChunkedDiscoverRows(&$data, $games, $title)
553 {
554     $chunks = array_chunk($games, 4);
555     $first = true;
556     foreach ($chunks as $chunk) {
557         addDiscoverRow(
558             $data, $first ? $title : '',
559             $chunk
560         );
561         $first = false;
562     }
563 }
564
565 function addDiscoverRow(&$data, $title, $games)
566 {
567     $row = [
568         'title'     => $title,
569         'showPrice' => true,
570         'ranked'    => false,
571         'tiles'     => [],
572     ];
573     foreach ($games as $game) {
574         if (is_string($game)) {
575             //category link
576             $tilePos = count($data['tiles']);
577             $data['tiles'][$tilePos] = buildDiscoverCategoryTile($game);
578
579         } else {
580             //game
581             if (isset($game->links->original)) {
582                 //do not link unlocked games.
583                 // people an access them via the original games
584                 continue;
585             }
586             $tilePos = findTile($data['tiles'], $game->packageName);
587             if ($tilePos === null) {
588                 $tilePos = count($data['tiles']);
589                 $data['tiles'][$tilePos] = buildDiscoverGameTile($game);
590             }
591         }
592         $row['tiles'][] = $tilePos;
593     }
594     $data['rows'][] = $row;
595 }
596
597 function findTile($tiles, $packageName)
598 {
599     foreach ($tiles as $pos => $tile) {
600         if ($tile['package'] == $packageName) {
601             return $pos;
602         }
603     }
604     return null;
605 }
606
607 function buildDiscoverCategoryTile($title)
608 {
609     return [
610         'url'   => 'ouya://launcher/discover/' . categoryPath($title),
611         'image' => '',
612         'title' => $title,
613         'type'  => 'discover'
614     ];
615 }
616
617 function buildDiscoverGameTile($game)
618 {
619     $latestRelease = $game->latestRelease;
620     return [
621         'gamerNumbers' => $game->players,
622         'genres' => $game->genres,
623         'url' => 'ouya://launcher/details?app=' . $game->packageName,
624         'latestVersion' => [
625             'apk' => [
626                 'md5sum' => $latestRelease->md5sum,
627             ],
628             'versionNumber' => $latestRelease->name,
629             'uuid' => $latestRelease->uuid,
630         ],
631         'inAppPurchases' => $game->inAppPurchases,
632         'promotedProduct' => null,
633         'premium' => $game->premium,
634         'type' => 'app',
635         'package' => $game->packageName,
636         'updated_at' => strtotime($latestRelease->date),
637         'updatedAt' => $latestRelease->date,
638         'title' => $game->title,
639         'image' => $game->discover,
640         'contentRating' => $game->contentRating,
641         'rating' => [
642             'count' => $game->rating->count,
643             'average' => $game->rating->average,
644         ],
645         'promotedProduct' => buildProduct(getPromotedProduct($game)),
646     ];
647 }
648
649 function categoryPath($title)
650 {
651     return str_replace(['/', '\\', ' ', '+', '?'], '_', $title);
652 }
653
654 function getAllAges($games)
655 {
656     $ages = [];
657     foreach ($games as $game) {
658         $ages[] = $game->contentRating;
659     }
660     return array_unique($ages);
661 }
662
663 function getAllGenres($games)
664 {
665     $genres = [];
666     foreach ($games as $game) {
667         $genres = array_merge($genres, $game->genres);
668     }
669     return array_unique($genres);
670 }
671
672 function addMissingGameProperties($game)
673 {
674     if (!isset($game->overview)) {
675         $game->overview = null;
676     }
677     if (!isset($game->description)) {
678         $game->description = '';
679     }
680     if (!isset($game->players)) {
681         $game->players = [1];
682     }
683     if (!isset($game->genres)) {
684         $game->genres = ['Unsorted'];
685     }
686     if (!isset($game->website)) {
687         $game->website = null;
688     }
689     if (!isset($game->contentRating)) {
690         $game->contentRating = 'Everyone';
691     }
692     if (!isset($game->premium)) {
693         $game->premium = false;
694     }
695     if (!isset($game->firstPublishedAt)) {
696         $game->firstPublishedAt = gmdate('c');
697     }
698
699     if (!isset($game->rating)) {
700         $game->rating = new stdClass();
701     }
702     if (!isset($game->rating->likeCount)) {
703         $game->rating->likeCount = 0;
704     }
705     if (!isset($game->rating->average)) {
706         $game->rating->average = 0;
707     }
708     if (!isset($game->rating->count)) {
709         $game->rating->count = 0;
710     }
711
712     $game->latestRelease = null;
713     $latestReleaseTimestamp = 0;
714     foreach ($game->releases as $release) {
715         if (!isset($release->publicSize)) {
716             $release->publicSize = 0;
717         }
718         if (!isset($release->nativeSize)) {
719             $release->nativeSize = 0;
720         }
721
722         $releaseTimestamp = strtotime($release->date);
723         if ($releaseTimestamp > $latestReleaseTimestamp) {
724             $game->latestRelease    = $release;
725             $latestReleaseTimestamp = $releaseTimestamp;
726         }
727     }
728     if ($game->latestRelease === null) {
729         error('No latest release for ' . $game->packageName);
730     }
731
732     if (!isset($game->media)) {
733         $game->media = [];
734     }
735
736     if (!isset($game->developer->uuid)) {
737         $game->developer->uuid = null;
738     }
739     if (!isset($game->developer->name)) {
740         $game->developer->name = 'unknown';
741     }
742     if (!isset($game->developer->supportEmail)) {
743         $game->developer->supportEmail = null;
744     }
745     if (!isset($game->developer->supportPhone)) {
746         $game->developer->supportPhone = null;
747     }
748     if (!isset($game->developer->founder)) {
749         $game->developer->founder = false;
750     }
751 }
752
753 /**
754  * Implements a sensible ranking system described in
755  * https://stackoverflow.com/a/1411268/2826013
756  */
757 function calculateRank(array $games)
758 {
759     $averageRatings = array_map(
760         function ($game) {
761             return $game->rating->average;
762         },
763         $games
764     );
765     $average = array_sum($averageRatings) / count($averageRatings);
766     $C = $average;
767     $m = 500;
768
769     foreach ($games as $game) {
770         $R = $game->rating->average;
771         $v = $game->rating->count;
772         $game->rating->rank = ($R * $v + $C * $m) / ($v + $m);
773     }
774 }
775
776 function getFirstVideoUrl($media)
777 {
778     foreach ($media as $medium) {
779         if ($medium->type == 'video') {
780             return $medium->url;
781         }
782     }
783     return null;
784 }
785
786 function getAllImageUrls($media)
787 {
788     $imageUrls = [];
789     foreach ($media as $medium) {
790         if ($medium->type == 'image') {
791             $imageUrls[] = $medium->url;
792         }
793     }
794     return $imageUrls;
795 }
796
797 function getPromotedProduct($game)
798 {
799     if (!isset($game->products) || !count($game->products)) {
800         return null;
801     }
802     foreach ($game->products as $gameProd) {
803         if ($gameProd->promoted) {
804             return $gameProd;
805         }
806     }
807     return null;
808 }
809
810 function removeMakeGames(array $games)
811 {
812     return filterByGenre($games, 'Tutorials', true);
813 }
814
815 function removeMakeGenres($genres)
816 {
817     $filtered = [];
818     foreach ($genres as $genre) {
819         if ($genre != 'Tutorials' && $genre != 'Builds') {
820             $filtered[] = $genre;
821         }
822     }
823     return $filtered;
824 }
825
826 function writeJson($path, $data)
827 {
828     global $wwwDir;
829     $fullPath = $wwwDir . $path;
830     $dir = dirname($fullPath);
831     if (!is_dir($dir)) {
832         mkdir($dir, 0777, true);
833     }
834     file_put_contents(
835         $fullPath,
836         json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"
837     );
838 }
839
840 function error($msg)
841 {
842     fwrite(STDERR, $msg . "\n");
843     exit(1);
844 }
845 ?>