fba427d6a97cd2b590d684d1cb37b5a6ad4a316b
[stouyapi.git] / bin / import-game-data.php
1 #!/usr/bin/env php
2 <?php
3 /**
4  * Import games from a OUYA game data repository
5  *
6  * @link https://github.com/cweiske/ouya-game-data/
7  * @author Christian Weiske <cweiske@cweiske.de>
8  */
9 ini_set('xdebug.halt_level', E_WARNING|E_NOTICE|E_USER_WARNING|E_USER_NOTICE);
10 require_once __DIR__ . '/filters.php';
11 if (!isset($argv[1])) {
12     error('Pass the path to a "folders" file with game data json files folder names');
13 }
14 $foldersFile = $argv[1];
15 if (!is_file($foldersFile)) {
16     error('Given path is not a file: ' . $foldersFile);
17 }
18
19 $GLOBALS['packagelists']['cweiskepicks'] = [
20     'de.eiswuxe.blookid2',
21     'com.cosmos.babyloniantwins',
22     'com.inverseblue.skyriders',
23 ];
24
25 $wwwDir = __DIR__ . '/../www/';
26
27 $baseDir   = dirname($foldersFile);
28 $gameFiles = [];
29 foreach (file($foldersFile) as $line) {
30     $line = trim($line);
31     if (strlen($line)) {
32         if (strpos($line, '..') !== false) {
33             error('Path attack in ' . $folder);
34         }
35         $folder = $baseDir . '/' . $line;
36         if (!is_dir($folder)) {
37             error('Folder does not exist: ' . $folder);
38         }
39         $gameFiles = array_merge($gameFiles, glob($folder . '/*.json'));
40     }
41 }
42
43 $games = [];
44 $count = 0;
45 $developers = [];
46 foreach ($gameFiles as $gameFile) {
47     $game = json_decode(file_get_contents($gameFile));
48     if ($game === null) {
49         error('JSON invalid at ' . $gameFile);
50     }
51     addMissingGameProperties($game);
52     $games[$game->packageName] = $game;
53
54     writeJson(
55         'api/v1/details-data/' . $game->packageName . '.json',
56         buildDetails($game)
57     );
58
59     if (!isset($developers[$game->developer->uuid])) {
60         $developers[$game->developer->uuid] = [
61             'info'     => $game->developer,
62             'products' => [],
63         ];
64     }
65
66     $products = $game->products ?? [];
67     foreach ($products as $product) {
68         writeJson(
69             'api/v1/developers/' . $game->developer->uuid
70             . '/products/' . $product->identifier . '.json',
71             buildDeveloperProductOnly($product, $game->developer)
72         );
73         $developers[$game->developer->uuid]['products'][] = $product;
74     }
75
76     /**/
77     writeJson(
78         'api/v1/games/' . $game->packageName . '/purchases',
79         buildPurchases($game)
80     );
81
82     writeJson(
83         'api/v1/apps/' . $game->packageName . '.json',
84         buildApps($game)
85     );
86     $latestRelease = $game->latestRelease;
87     writeJson(
88         'api/v1/apps/' . $latestRelease->uuid . '.json',
89         buildApps($game)
90     );
91
92     writeJson(
93         'api/v1/apps/' . $latestRelease->uuid . '-download.json',
94         buildAppDownload($game, $latestRelease)
95     );
96
97     if ($count++ > 20) {
98         //break;
99     }
100 }
101
102 calculateRank($games);
103
104 foreach ($developers as $developer) {
105     writeJson(
106         //index.htm does not need a rewrite rule
107         'api/v1/developers/' . $developer['info']->uuid
108         . '/products/index.htm',
109         buildDeveloperProducts($developer['products'], $developer['info'])
110     );
111     writeJson(
112         //index.htm does not need a rewrite rule
113         'api/v1/developers/' . $developer['info']->uuid
114         . '/current_gamer',
115         buildDeveloperCurrentGamer()
116     );
117 }
118
119 writeJson('api/v1/discover-data/discover.json', buildDiscover($games));
120 writeJson('api/v1/discover-data/home.json', buildDiscoverHome($games));
121
122 //make
123 writeJson(
124     'api/v1/discover-data/tutorials.json',
125     buildMakeCategory('Tutorials', filterByGenre($games, 'Tutorials'))
126 );
127
128 $searchLetters = 'abcdefghijklmnopqrstuvwxyz0123456789., ';
129 foreach (str_split($searchLetters) as $letter) {
130     $letterGames = filterBySearchWord($games, $letter);
131     writeJson(
132         'api/v1/search-data/' . $letter . '.json',
133         buildSearch($letterGames)
134     );
135 }
136
137
138 function buildDiscover(array $games)
139 {
140     $games = removeMakeGames($games);
141     $data = [
142         'title' => 'DISCOVER',
143         'rows'  => [],
144         'tiles' => [],
145     ];
146
147     addDiscoverRow(
148         $data, 'Last Updated',
149         filterLastUpdated($games, 10)
150     );
151     addDiscoverRow(
152         $data, 'Best rated',
153         filterBestRated($games, 10)
154     );
155     addDiscoverRow(
156         $data, "cweiske's picks",
157         filterByPackageNames($games, $GLOBALS['packagelists']['cweiskepicks'])
158     );
159
160     $players = [
161         //1 => '1 player',
162         2 => '2 players',
163         3 => '3 players',
164         4 => '4 players',
165     ];
166     addDiscoverRow($data, '# of players', $players);
167     foreach ($players as $num => $title) {
168         writeJson(
169             'api/v1/discover-data/' . categoryPath($title) . '.json',
170             buildDiscoverCategory($title, filterByPlayers($games, $num))
171         );
172     }
173
174     $ages = getAllAges($games);
175     natsort($ages);
176     addDiscoverRow($data, 'Content rating', $ages);
177     foreach ($ages as $num => $title) {
178         writeJson(
179             'api/v1/discover-data/' . categoryPath($title) . '.json',
180             buildDiscoverCategory($title, filterByAge($games, $title))
181         );
182     }
183
184     $genres = removeMakeGenres(getAllGenres($games));
185     sort($genres);
186     addChunkedDiscoverRows($data, $genres, 'Genres');
187
188     foreach ($genres as $genre) {
189         writeJson(
190             'api/v1/discover-data/' . categoryPath($genre) . '.json',
191             buildDiscoverCategory($genre, filterByGenre($games, $genre))
192         );
193     }
194
195     $abc = array_merge(range('A', 'Z'), ['Other']);
196     addChunkedDiscoverRows($data, $abc, 'Alphabetical');
197     foreach ($abc as $letter) {
198         writeJson(
199             'api/v1/discover-data/' . categoryPath($letter) . '.json',
200             buildDiscoverCategory($letter, filterByLetter($games, $letter))
201         );
202     }
203
204     return $data;
205 }
206
207 /**
208  * A genre category page
209  */
210 function buildDiscoverCategory($name, $games)
211 {
212     $data = [
213         'title' => $name,
214         'rows'  => [],
215         'tiles' => [],
216     ];
217     addDiscoverRow(
218         $data, 'Last Updated',
219         filterLastUpdated($games, 10)
220     );
221     addDiscoverRow(
222         $data, 'Best rated',
223         filterBestRated($games, 10)
224     );
225
226     $games = sortByTitle($games);
227     $chunks = array_chunk($games, 4);
228     foreach ($chunks as $chunkGames) {
229         addDiscoverRow($data, '', $chunkGames);
230     }
231
232     return $data;
233 }
234
235 function buildMakeCategory($name, $games)
236 {
237     $data = [
238         'title' => $name,
239         'rows'  => [],
240         'tiles' => [],
241     ];
242
243     $games = sortByTitle($games);
244     addDiscoverRow($data, '', $games);
245
246     return $data;
247 }
248
249 function buildDiscoverHome(array $games)
250 {
251     //we do not want anything here for now
252     $data = [
253         'title' => 'home',
254         'rows'  => [
255             [
256                 'title' => 'FEATURED',
257                 'showPrice' => false,
258                 'ranked'    => false,
259                 'tiles'     => [],
260             ]
261         ],
262         'tiles' => [],
263     ];
264     return $data;
265 }
266
267 /**
268  * Build api/v1/apps/$packageName
269  */
270 function buildApps($game)
271 {
272     $latestRelease = $game->latestRelease;
273
274     $product      = null;
275     $gamePromoted = getPromotedProduct($game);
276     if ($gamePromoted) {
277         $product = buildProduct($gamePromoted);
278     }
279
280     // http://cweiske.de/ouya-store-api-docs.htm#get-https-devs-ouya-tv-api-v1-apps-xxx
281     return [
282         'app' => [
283             'uuid'          => $latestRelease->uuid,
284             'title'         => $game->title,
285             'overview'      => $game->overview,
286             'description'   => $game->description,
287             'gamerNumbers'  => $game->players,
288             'genres'        => $game->genres,
289
290             'website'       => $game->website,
291             'contentRating' => $game->contentRating,
292             'premium'       => $game->premium,
293             'firstPublishedAt' => $game->firstPublishedAt,
294
295             'likeCount'     => $game->rating->likeCount,
296             'ratingAverage' => $game->rating->average,
297             'ratingCount'   => $game->rating->count,
298
299             'versionNumber' => $latestRelease->name,
300             'latestVersion' => $latestRelease->uuid,
301             'md5sum'        => $latestRelease->md5sum,
302             'apkFileSize'   => $latestRelease->size,
303             'publishedAt'   => $latestRelease->date,
304             'publicSize'    => $latestRelease->publicSize,
305             'nativeSize'    => $latestRelease->nativeSize,
306
307             'mainImageFullUrl' => $game->discover,
308             'videoUrl'         => getFirstVideoUrl($game->media),
309             'filepickerScreenshots' => getAllImageUrls($game->media),
310             'mobileAppIcon'    => null,
311
312             'developer'           => $game->developer->name,
313             'supportEmailAddress' => $game->developer->supportEmail,
314             'supportPhone'        => $game->developer->supportPhone,
315             'founder'             => $game->developer->founder,
316
317             'promotedProduct' => $product,
318         ],
319     ];
320 }
321
322 function buildAppDownload($game, $release)
323 {
324     return [
325         'app' => [
326             'fileSize'      => $release->size,
327             'version'       => $release->uuid,
328             'contentRating' => $game->contentRating,
329             'downloadLink'  => $release->url,
330         ]
331     ];
332 }
333
334 function buildProduct($product)
335 {
336     if ($product === null) {
337         return null;
338     }
339     return [
340         'type'          => 'entitlement',
341         'identifier'    => $product->identifier,
342         'name'          => $product->name,
343         'description'   => $product->description ?? '',
344         'localPrice'    => $product->localPrice,
345         'originalPrice' => $product->originalPrice,
346         'percentOff'    => 0,
347         'currency'      => $product->currency,
348     ];
349 }
350
351 /**
352  * Build /app/v1/details?app=org.example.game
353  */
354 function buildDetails($game)
355 {
356     $latestRelease = $game->latestRelease;
357
358     $mediaTiles = [];
359     if ($game->discover) {
360         $mediaTiles[] = [
361             'type' => 'image',
362             'urls' => [
363                 'thumbnail' => $game->discover,
364                 'full'      => $game->discover,
365             ],
366         ];
367     }
368     foreach ($game->media as $medium) {
369         if ($medium->type == 'image')  {
370             $mediaTiles[] = [
371                 'type' => 'image',
372                 'urls' => [
373                     'thumbnail' => $medium->thumb,
374                     'full'      => $medium->url,
375                 ],
376             ];
377         } else {
378             $mediaTiles[] = [
379                 'type' => 'video',
380                 'url'  => $medium->url,
381             ];
382         }
383     }
384
385     $buttons = [];
386     if (isset($game->links->unlocked)) {
387         $buttons[] = [
388             'text' => 'Show unlocked',
389             'url'  => 'ouya://launcher/details?app=' . $game->links->unlocked,
390             'bold' => true,
391         ];
392     }
393
394     $product      = null;
395     $gamePromoted = getPromotedProduct($game);
396     if ($gamePromoted) {
397         $product = buildProduct($gamePromoted);
398     }
399
400     // http://cweiske.de/ouya-store-api-docs.htm#get-https-devs-ouya-tv-api-v1-details
401     return [
402         'type'             => 'Game',
403         'title'            => $game->title,
404         'description'      => $game->description,
405         'gamerNumbers'     => $game->players,
406         'genres'           => $game->genres,
407
408         'suggestedAge'     => $game->contentRating,
409         'premium'          => $game->premium,
410         'inAppPurchases'   => $game->inAppPurchases,
411         'firstPublishedAt' => strtotime($game->firstPublishedAt),
412         'ccUrl'            => null,
413
414         'rating' => [
415             'count'   => $game->rating->count,
416             'average' => $game->rating->average,
417         ],
418
419         'apk' => [
420             'fileSize'    => $latestRelease->size,
421             'nativeSize'  => $latestRelease->nativeSize,
422             'publicSize'  => $latestRelease->publicSize,
423             'md5sum'      => $latestRelease->md5sum,
424             'filename'    => 'FIXME',
425             'errors'      => '',
426             'package'     => $game->packageName,
427             'versionCode' => $latestRelease->versionCode,
428             'state'       => 'complete',
429         ],
430
431         'version' => [
432             'number'      => $latestRelease->name,
433             'publishedAt' => strtotime($latestRelease->date),
434             'uuid'        => $latestRelease->uuid,
435         ],
436
437         'developer' => [
438             'name'    => $game->developer->name,
439             'founder' => $game->developer->founder,
440         ],
441
442         'metaData' => [
443             'key:rating.average',
444             'key:developer.name',
445             'key:suggestedAge',
446             number_format($latestRelease->size / 1024 / 1024, 2, '.', '') . ' MiB',
447         ],
448
449         'tileImage'     => $game->discover,
450         'mediaTiles'    => $mediaTiles,
451         'mobileAppIcon' => null,
452         'heroImage'     => [
453             'url' => null,
454         ],
455
456         'promotedProduct' => $product,
457         'buttons'         => $buttons,
458     ];
459 }
460
461 function buildDeveloperCurrentGamer()
462 {
463     return [
464         'gamer' => [
465             'uuid'     => '00702342-0000-1111-2222-c3e1500cafe2',
466             'username' => 'stouyapi',
467         ],
468     ];
469 }
470
471 /**
472  * For /api/v1/developers/xxx/products/?only=yyy
473  */
474 function buildDeveloperProductOnly($product, $developer)
475 {
476     return [
477         'developerName' => $developer->name,
478         'currency'      => $product->currency,
479         'products'      => [
480             buildProduct($product),
481         ],
482     ];
483 }
484
485 /**
486  * For /api/v1/developers/xxx/products/
487  */
488 function buildDeveloperProducts($products, $developer)
489 {
490     $jsonProducts = [];
491     foreach ($products as $product) {
492         $jsonProducts[] = buildProduct($product);
493     }
494     return [
495         'developerName' => $developer->name,
496         'currency'      => $products[0]->currency ?? 'EUR',
497         'products'      => $jsonProducts,
498     ];
499 }
500
501 function buildPurchases($game)
502 {
503     $purchasesData = [
504         'purchases' => [],
505     ];
506     $promotedProduct = getPromotedProduct($game);
507     if ($promotedProduct) {
508         $purchasesData['purchases'][] = [
509             'purchaseDate' => time() * 1000,
510             'generateDate' => time() * 1000,
511             'identifier'   => $promotedProduct->identifier,
512             'gamer'        => '00702342-0000-1111-2222-c3e1500cafe2',//gamer uuid
513             'uuid'         => '00702342-0000-1111-2222-c3e1500beef3',//transaction ID
514             'priceInCents' => $promotedProduct->originalPrice * 100,
515             'localPrice'   => $promotedProduct->localPrice,
516             'currency'     => $promotedProduct->currency,
517         ];
518     }
519
520     $encryptedOnce  = dummyEncrypt($purchasesData);
521     $encryptedTwice = dummyEncrypt($encryptedOnce);
522     return $encryptedTwice;
523 }
524
525 function buildSearch($games)
526 {
527     $games = sortByTitle($games);
528     $results = [];
529     foreach ($games as $game) {
530         $results[] = [
531             'title' => $game->title,
532             'url'   => 'ouya://launcher/details?app=' . $game->packageName,
533             'contentRating' => $game->contentRating,
534         ];
535     }
536     return [
537         'count'   => count($results),
538         'results' => $results,
539     ];
540 }
541
542 function dummyEncrypt($data)
543 {
544     return [
545         'key'  => base64_encode('0123456789abcdef') . "\n",
546         'iv'   => 't3jir1LHpICunvhlM76edQ==' . "\n",//random bytes
547         'blob' => base64_encode(
548             json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)
549         ) . "\n",
550     ];
551 }
552
553 function addChunkedDiscoverRows(&$data, $games, $title)
554 {
555     $chunks = array_chunk($games, 4);
556     $first = true;
557     foreach ($chunks as $chunk) {
558         addDiscoverRow(
559             $data, $first ? $title : '',
560             $chunk
561         );
562         $first = false;
563     }
564 }
565
566 function addDiscoverRow(&$data, $title, $games)
567 {
568     $row = [
569         'title'     => $title,
570         'showPrice' => true,
571         'ranked'    => false,
572         'tiles'     => [],
573     ];
574     foreach ($games as $game) {
575         if (is_string($game)) {
576             //category link
577             $tilePos = count($data['tiles']);
578             $data['tiles'][$tilePos] = buildDiscoverCategoryTile($game);
579
580         } else {
581             //game
582             if (isset($game->links->original)) {
583                 //do not link unlocked games.
584                 // people an access them via the original games
585                 continue;
586             }
587             $tilePos = findTile($data['tiles'], $game->packageName);
588             if ($tilePos === null) {
589                 $tilePos = count($data['tiles']);
590                 $data['tiles'][$tilePos] = buildDiscoverGameTile($game);
591             }
592         }
593         $row['tiles'][] = $tilePos;
594     }
595     $data['rows'][] = $row;
596 }
597
598 function findTile($tiles, $packageName)
599 {
600     foreach ($tiles as $pos => $tile) {
601         if ($tile['package'] == $packageName) {
602             return $pos;
603         }
604     }
605     return null;
606 }
607
608 function buildDiscoverCategoryTile($title)
609 {
610     return [
611         'url'   => 'ouya://launcher/discover/' . categoryPath($title),
612         'image' => '',
613         'title' => $title,
614         'type'  => 'discover'
615     ];
616 }
617
618 function buildDiscoverGameTile($game)
619 {
620     $latestRelease = $game->latestRelease;
621     return [
622         'gamerNumbers' => $game->players,
623         'genres' => $game->genres,
624         'url' => 'ouya://launcher/details?app=' . $game->packageName,
625         'latestVersion' => [
626             'apk' => [
627                 'md5sum' => $latestRelease->md5sum,
628             ],
629             'versionNumber' => $latestRelease->name,
630             'uuid' => $latestRelease->uuid,
631         ],
632         'inAppPurchases' => $game->inAppPurchases,
633         'promotedProduct' => null,
634         'premium' => $game->premium,
635         'type' => 'app',
636         'package' => $game->packageName,
637         'updated_at' => strtotime($latestRelease->date),
638         'updatedAt' => $latestRelease->date,
639         'title' => $game->title,
640         'image' => $game->discover,
641         'contentRating' => $game->contentRating,
642         'rating' => [
643             'count' => $game->rating->count,
644             'average' => $game->rating->average,
645         ],
646         'promotedProduct' => buildProduct(getPromotedProduct($game)),
647     ];
648 }
649
650 function categoryPath($title)
651 {
652     return str_replace(['/', '\\', ' ', '+', '?'], '_', $title);
653 }
654
655 function getAllAges($games)
656 {
657     $ages = [];
658     foreach ($games as $game) {
659         $ages[] = $game->contentRating;
660     }
661     return array_unique($ages);
662 }
663
664 function getAllGenres($games)
665 {
666     $genres = [];
667     foreach ($games as $game) {
668         $genres = array_merge($genres, $game->genres);
669     }
670     return array_unique($genres);
671 }
672
673 function addMissingGameProperties($game)
674 {
675     if (!isset($game->overview)) {
676         $game->overview = null;
677     }
678     if (!isset($game->description)) {
679         $game->description = '';
680     }
681     if (!isset($game->players)) {
682         $game->players = [1];
683     }
684     if (!isset($game->genres)) {
685         $game->genres = ['Unsorted'];
686     }
687     if (!isset($game->website)) {
688         $game->website = null;
689     }
690     if (!isset($game->contentRating)) {
691         $game->contentRating = 'Everyone';
692     }
693     if (!isset($game->premium)) {
694         $game->premium = false;
695     }
696     if (!isset($game->firstPublishedAt)) {
697         $game->firstPublishedAt = gmdate('c');
698     }
699
700     if (!isset($game->rating)) {
701         $game->rating = new stdClass();
702     }
703     if (!isset($game->rating->likeCount)) {
704         $game->rating->likeCount = 0;
705     }
706     if (!isset($game->rating->average)) {
707         $game->rating->average = 0;
708     }
709     if (!isset($game->rating->count)) {
710         $game->rating->count = 0;
711     }
712
713     $game->latestRelease = null;
714     $latestReleaseTimestamp = 0;
715     foreach ($game->releases as $release) {
716         if (!isset($release->publicSize)) {
717             $release->publicSize = 0;
718         }
719         if (!isset($release->nativeSize)) {
720             $release->nativeSize = 0;
721         }
722
723         $releaseTimestamp = strtotime($release->date);
724         if ($releaseTimestamp > $latestReleaseTimestamp) {
725             $game->latestRelease    = $release;
726             $latestReleaseTimestamp = $releaseTimestamp;
727         }
728     }
729     if ($game->latestRelease === null) {
730         error('No latest release for ' . $game->packageName);
731     }
732
733     if (!isset($game->media)) {
734         $game->media = [];
735     }
736
737     if (!isset($game->developer->uuid)) {
738         $game->developer->uuid = null;
739     }
740     if (!isset($game->developer->name)) {
741         $game->developer->name = 'unknown';
742     }
743     if (!isset($game->developer->supportEmail)) {
744         $game->developer->supportEmail = null;
745     }
746     if (!isset($game->developer->supportPhone)) {
747         $game->developer->supportPhone = null;
748     }
749     if (!isset($game->developer->founder)) {
750         $game->developer->founder = false;
751     }
752 }
753
754 /**
755  * Implements a sensible ranking system described in
756  * https://stackoverflow.com/a/1411268/2826013
757  */
758 function calculateRank(array $games)
759 {
760     $averageRatings = array_map(
761         function ($game) {
762             return $game->rating->average;
763         },
764         $games
765     );
766     $average = array_sum($averageRatings) / count($averageRatings);
767     $C = $average;
768     $m = 500;
769
770     foreach ($games as $game) {
771         $R = $game->rating->average;
772         $v = $game->rating->count;
773         $game->rating->rank = ($R * $v + $C * $m) / ($v + $m);
774     }
775 }
776
777 function getFirstVideoUrl($media)
778 {
779     foreach ($media as $medium) {
780         if ($medium->type == 'video') {
781             return $medium->url;
782         }
783     }
784     return null;
785 }
786
787 function getAllImageUrls($media)
788 {
789     $imageUrls = [];
790     foreach ($media as $medium) {
791         if ($medium->type == 'image') {
792             $imageUrls[] = $medium->url;
793         }
794     }
795     return $imageUrls;
796 }
797
798 function getPromotedProduct($game)
799 {
800     if (!isset($game->products) || !count($game->products)) {
801         return null;
802     }
803     foreach ($game->products as $gameProd) {
804         if ($gameProd->promoted) {
805             return $gameProd;
806         }
807     }
808     return null;
809 }
810
811 function removeMakeGames(array $games)
812 {
813     return filterByGenre($games, 'Tutorials', true);
814 }
815
816 function removeMakeGenres($genres)
817 {
818     $filtered = [];
819     foreach ($genres as $genre) {
820         if ($genre != 'Tutorials' && $genre != 'Builds') {
821             $filtered[] = $genre;
822         }
823     }
824     return $filtered;
825 }
826
827 function writeJson($path, $data)
828 {
829     global $wwwDir;
830     $fullPath = $wwwDir . $path;
831     $dir = dirname($fullPath);
832     if (!is_dir($dir)) {
833         mkdir($dir, 0777, true);
834     }
835     file_put_contents(
836         $fullPath,
837         json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"
838     );
839 }
840
841 function error($msg)
842 {
843     fwrite(STDERR, $msg . "\n");
844     exit(1);
845 }
846 ?>