much better ranking
[stouyapi.git] / bin / import-game-data.php
1 #!/usr/bin/env php
2 <?php
3 /**
4  * Import games from a OUYA game data repository
5  *
6  * @link https://github.com/cweiske/ouya-game-data/
7  * @author Christian Weiske <cweiske@cweiske.de>
8  */
9 ini_set('xdebug.halt_level', E_WARNING|E_NOTICE|E_USER_WARNING|E_USER_NOTICE);
10 require_once __DIR__ . '/filters.php';
11 if (!isset($argv[1])) {
12     error('Pass the path to a "folders" file with game data json files folder names');
13 }
14 $foldersFile = $argv[1];
15 if (!is_file($foldersFile)) {
16     error('Given path is not a file: ' . $foldersFile);
17 }
18
19 $GLOBALS['packagelists']['cweiskepicks'] = [
20     'de.eiswuxe.blookid2',
21     'com.cosmos.babyloniantwins'
22 ];
23
24 $wwwDir = __DIR__ . '/../www/';
25
26 $baseDir   = dirname($foldersFile);
27 $gameFiles = [];
28 foreach (file($foldersFile) as $line) {
29     $line = trim($line);
30     if (strlen($line)) {
31         if (strpos($line, '..') !== false) {
32             error('Path attack in ' . $folder);
33         }
34         $folder = $baseDir . '/' . $line;
35         if (!is_dir($folder)) {
36             error('Folder does not exist: ' . $folder);
37         }
38         $gameFiles = array_merge($gameFiles, glob($folder . '/*.json'));
39     }
40 }
41
42 $games = [];
43 $count = 0;
44 $developers = [];
45 foreach ($gameFiles as $gameFile) {
46     $game = json_decode(file_get_contents($gameFile));
47     if ($game === null) {
48         error('JSON invalid at ' . $gameFile);
49     }
50     addMissingGameProperties($game);
51     $games[$game->packageName] = $game;
52
53     writeJson(
54         'api/v1/details-data/' . $game->packageName . '.json',
55         buildDetails($game)
56     );
57
58     if (!isset($developers[$game->developer->uuid])) {
59         $developers[$game->developer->uuid] = [
60             'info'     => $game->developer,
61             'products' => [],
62         ];
63     }
64
65     $products = $game->products ?? [];
66     foreach ($products as $product) {
67         writeJson(
68             'api/v1/developers/' . $game->developer->uuid
69             . '/products/' . $product->identifier . '.json',
70             buildDeveloperProductOnly($product, $game->developer)
71         );
72         $developers[$game->developer->uuid]['products'][] = $product;
73     }
74
75     /**/
76     writeJson(
77         'api/v1/games/' . $game->packageName . '/purchases',
78         buildPurchases($game)
79     );
80
81     writeJson(
82         'api/v1/apps/' . $game->packageName . '.json',
83         buildApps($game)
84     );
85     $latestRelease = $game->latestRelease;
86     writeJson(
87         'api/v1/apps/' . $latestRelease->uuid . '.json',
88         buildApps($game)
89     );
90
91     writeJson(
92         'api/v1/apps/' . $latestRelease->uuid . '-download.json',
93         buildAppDownload($game, $latestRelease)
94     );
95
96     if ($count++ > 20) {
97         //break;
98     }
99 }
100
101 calculateRank($games);
102
103 foreach ($developers as $developer) {
104     writeJson(
105         //index.htm does not need a rewrite rule
106         'api/v1/developers/' . $developer['info']->uuid
107         . '/products/index.htm',
108         buildDeveloperProducts($developer['products'], $developer['info'])
109     );
110     writeJson(
111         //index.htm does not need a rewrite rule
112         'api/v1/developers/' . $developer['info']->uuid
113         . '/current_gamer',
114         buildDeveloperCurrentGamer()
115     );
116 }
117
118 writeJson('api/v1/discover-data/discover.json', buildDiscover($games));
119 writeJson('api/v1/discover-data/home.json', buildDiscoverHome($games));
120
121 //make
122 writeJson(
123     'api/v1/discover-data/tutorials.json',
124     buildMakeCategory('Tutorials', filterByGenre($games, 'Tutorials'))
125 );
126
127
128
129 function buildDiscover(array $games)
130 {
131     $games = removeMakeGames($games);
132     $data = [
133         'title' => 'DISCOVER',
134         'rows'  => [],
135         'tiles' => [],
136     ];
137
138     addDiscoverRow(
139         $data, 'Last Updated',
140         filterLastUpdated($games, 10)
141     );
142     addDiscoverRow(
143         $data, 'Best rated',
144         filterBestRated($games, 10)
145     );
146     addDiscoverRow(
147         $data, "cweiske's picks",
148         filterByPackageNames($games, $GLOBALS['packagelists']['cweiskepicks'])
149     );
150
151     $players = [
152         //1 => '1 player',
153         2 => '2 players',
154         3 => '3 players',
155         4 => '4 players',
156     ];
157     addDiscoverRow($data, '# of players', $players);
158     foreach ($players as $num => $title) {
159         writeJson(
160             'api/v1/discover-data/' . categoryPath($title) . '.json',
161             buildDiscoverCategory($title, filterByPlayers($games, $num))
162         );
163     }
164
165     $ages = getAllAges($games);
166     natsort($ages);
167     addDiscoverRow($data, 'Content rating', $ages);
168     foreach ($ages as $num => $title) {
169         writeJson(
170             'api/v1/discover-data/' . categoryPath($title) . '.json',
171             buildDiscoverCategory($title, filterByAge($games, $title))
172         );
173     }
174
175     $genres = removeMakeGenres(getAllGenres($games));
176     sort($genres);
177     addChunkedDiscoverRows($data, $genres, 'Genres');
178
179     foreach ($genres as $genre) {
180         writeJson(
181             'api/v1/discover-data/' . categoryPath($genre) . '.json',
182             buildDiscoverCategory($genre, filterByGenre($games, $genre))
183         );
184     }
185
186     $abc = array_merge(range('A', 'Z'), ['Other']);
187     addChunkedDiscoverRows($data, $abc, 'Alphabetical');
188     foreach ($abc as $letter) {
189         writeJson(
190             'api/v1/discover-data/' . categoryPath($letter) . '.json',
191             buildDiscoverCategory($letter, filterByLetter($games, $letter))
192         );
193     }
194
195     return $data;
196 }
197
198 /**
199  * A genre category page
200  */
201 function buildDiscoverCategory($name, $games)
202 {
203     $data = [
204         'title' => $name,
205         'rows'  => [],
206         'tiles' => [],
207     ];
208     addDiscoverRow(
209         $data, 'Last Updated',
210         filterLastUpdated($games, 10)
211     );
212     addDiscoverRow(
213         $data, 'Best rated',
214         filterBestRated($games, 10)
215     );
216
217     usort(
218         $games,
219         function ($gameA, $gameB) {
220             return strcasecmp($gameA->title, $gameB->title);
221         }
222     );
223     $chunks = array_chunk($games, 4);
224     foreach ($chunks as $chunkGames) {
225         addDiscoverRow($data, '', $chunkGames);
226     }
227
228     return $data;
229 }
230
231 function buildMakeCategory($name, $games)
232 {
233     $data = [
234         'title' => $name,
235         'rows'  => [],
236         'tiles' => [],
237     ];
238
239     usort(
240         $games,
241         function ($gameA, $gameB) {
242             return strcasecmp($gameA->title, $gameB->title);
243         }
244     );
245     addDiscoverRow($data, '', $games);
246
247     return $data;
248 }
249
250 function buildDiscoverHome(array $games)
251 {
252     //we do not want anything here for now
253     $data = [
254         'title' => 'home',
255         'rows'  => [
256             [
257                 'title' => 'FEATURED',
258                 'showPrice' => false,
259                 'ranked'    => false,
260                 'tiles'     => [],
261             ]
262         ],
263         'tiles' => [],
264     ];
265     return $data;
266 }
267
268 /**
269  * Build api/v1/apps/$packageName
270  */
271 function buildApps($game)
272 {
273     $latestRelease = $game->latestRelease;
274
275     $product      = null;
276     $gamePromoted = getPromotedProduct($game);
277     if ($gamePromoted) {
278         $product = buildProduct($gamePromoted);
279     }
280
281     // http://cweiske.de/ouya-store-api-docs.htm#get-https-devs-ouya-tv-api-v1-apps-xxx
282     return [
283         'app' => [
284             'uuid'          => $latestRelease->uuid,
285             'title'         => $game->title,
286             'overview'      => $game->overview,
287             'description'   => $game->description,
288             'gamerNumbers'  => $game->players,
289             'genres'        => $game->genres,
290
291             'website'       => $game->website,
292             'contentRating' => $game->contentRating,
293             'premium'       => $game->premium,
294             'firstPublishedAt' => $game->firstPublishedAt,
295
296             'likeCount'     => $game->rating->likeCount,
297             'ratingAverage' => $game->rating->average,
298             'ratingCount'   => $game->rating->count,
299
300             'versionNumber' => $latestRelease->name,
301             'latestVersion' => $latestRelease->uuid,
302             'md5sum'        => $latestRelease->md5sum,
303             'apkFileSize'   => $latestRelease->size,
304             'publishedAt'   => $latestRelease->date,
305             'publicSize'    => $latestRelease->publicSize,
306             'nativeSize'    => $latestRelease->nativeSize,
307
308             'mainImageFullUrl' => $game->discover,
309             'videoUrl'         => getFirstVideoUrl($game->media),
310             'filepickerScreenshots' => getAllImageUrls($game->media),
311             'mobileAppIcon'    => null,
312
313             'developer'           => $game->developer->name,
314             'supportEmailAddress' => $game->developer->supportEmail,
315             'supportPhone'        => $game->developer->supportPhone,
316             'founder'             => $game->developer->founder,
317
318             'promotedProduct' => $product,
319         ],
320     ];
321 }
322
323 function buildAppDownload($game, $release)
324 {
325     return [
326         'app' => [
327             'fileSize'      => $release->size,
328             'version'       => $release->uuid,
329             'contentRating' => $game->contentRating,
330             'downloadLink'  => $release->url,
331         ]
332     ];
333 }
334
335 function buildProduct($product)
336 {
337     if ($product === null) {
338         return null;
339     }
340     return [
341         'type'          => 'entitlement',
342         'identifier'    => $product->identifier,
343         'name'          => $product->name,
344         'description'   => $product->description ?? '',
345         'localPrice'    => $product->localPrice,
346         'originalPrice' => $product->originalPrice,
347         'percentOff'    => 0,
348         'currency'      => $product->currency,
349     ];
350 }
351
352 /**
353  * Build /app/v1/details?app=org.example.game
354  */
355 function buildDetails($game)
356 {
357     $latestRelease = $game->latestRelease;
358
359     $mediaTiles = [];
360     if ($game->discover) {
361         $mediaTiles[] = [
362             'type' => 'image',
363             'urls' => [
364                 'thumbnail' => $game->discover,
365                 'full'      => $game->discover,
366             ],
367         ];
368     }
369     foreach ($game->media as $medium) {
370         if ($medium->type == 'image')  {
371             $mediaTiles[] = [
372                 'type' => 'image',
373                 'urls' => [
374                     'thumbnail' => $medium->thumb,
375                     'full'      => $medium->url,
376                 ],
377             ];
378         } else {
379             $mediaTiles[] = [
380                 'type' => 'video',
381                 'url'  => $medium->url,
382             ];
383         }
384     }
385
386     $buttons = [];
387     if (isset($game->links->unlocked)) {
388         $buttons[] = [
389             'text' => 'Show unlocked',
390             'url'  => 'ouya://launcher/details?app=' . $game->links->unlocked,
391             'bold' => true,
392         ];
393     }
394
395     $product      = null;
396     $gamePromoted = getPromotedProduct($game);
397     if ($gamePromoted) {
398         $product = buildProduct($gamePromoted);
399     }
400
401     // http://cweiske.de/ouya-store-api-docs.htm#get-https-devs-ouya-tv-api-v1-details
402     return [
403         'type'             => 'Game',
404         'title'            => $game->title,
405         'description'      => $game->description,
406         'gamerNumbers'     => $game->players,
407         'genres'           => $game->genres,
408
409         'suggestedAge'     => $game->contentRating,
410         'premium'          => $game->premium,
411         'inAppPurchases'   => $game->inAppPurchases,
412         'firstPublishedAt' => strtotime($game->firstPublishedAt),
413         'ccUrl'            => null,
414
415         'rating' => [
416             'count'   => $game->rating->count,
417             'average' => $game->rating->average,
418         ],
419
420         'apk' => [
421             'fileSize'    => $latestRelease->size,
422             'nativeSize'  => $latestRelease->nativeSize,
423             'publicSize'  => $latestRelease->publicSize,
424             'md5sum'      => $latestRelease->md5sum,
425             'filename'    => 'FIXME',
426             'errors'      => '',
427             'package'     => $game->packageName,
428             'versionCode' => $latestRelease->versionCode,
429             'state'       => 'complete',
430         ],
431
432         'version' => [
433             'number'      => $latestRelease->name,
434             'publishedAt' => strtotime($latestRelease->date),
435             'uuid'        => $latestRelease->uuid,
436         ],
437
438         'developer' => [
439             'name'    => $game->developer->name,
440             'founder' => $game->developer->founder,
441         ],
442
443         'metaData' => [
444             'key:rating.average',
445             'key:developer.name',
446             'key:suggestedAge',
447             number_format($latestRelease->size / 1024 / 1024, 2, '.', '') . ' MiB',
448         ],
449
450         'tileImage'     => $game->discover,
451         'mediaTiles'    => $mediaTiles,
452         'mobileAppIcon' => null,
453         'heroImage'     => [
454             'url' => null,
455         ],
456
457         'promotedProduct' => $product,
458         'buttons'         => $buttons,
459     ];
460 }
461
462 function buildDeveloperCurrentGamer()
463 {
464     return [
465         'gamer' => [
466             'uuid'     => '00702342-0000-1111-2222-c3e1500cafe2',
467             'username' => 'stouyapi',
468         ],
469     ];
470 }
471
472 /**
473  * For /api/v1/developers/xxx/products/?only=yyy
474  */
475 function buildDeveloperProductOnly($product, $developer)
476 {
477     return [
478         'developerName' => $developer->name,
479         'currency'      => $product->currency,
480         'products'      => [
481             buildProduct($product),
482         ],
483     ];
484 }
485
486 /**
487  * For /api/v1/developers/xxx/products/
488  */
489 function buildDeveloperProducts($products, $developer)
490 {
491     $jsonProducts = [];
492     foreach ($products as $product) {
493         $jsonProducts[] = buildProduct($product);
494     }
495     return [
496         'developerName' => $developer->name,
497         'currency'      => $products[0]->currency ?? 'EUR',
498         'products'      => $jsonProducts,
499     ];
500 }
501
502 function buildPurchases($game)
503 {
504     $purchasesData = [
505         'purchases' => [],
506     ];
507     $promotedProduct = getPromotedProduct($game);
508     if ($promotedProduct) {
509         $purchasesData['purchases'][] = [
510             'purchaseDate' => time() * 1000,
511             'generateDate' => time() * 1000,
512             'identifier'   => $promotedProduct->identifier,
513             'gamer'        => 'stouyapi',
514             'uuid'         => '00702342-0000-1111-2222-c3e1500cafe2',//gamer uuid
515             'priceInCents' => $promotedProduct->originalPrice * 100,
516             'localPrice'   => $promotedProduct->localPrice,
517             'currency'     => $promotedProduct->currency,
518         ];
519     }
520
521     $encryptedOnce  = dummyEncrypt($purchasesData);
522     $encryptedTwice = dummyEncrypt($encryptedOnce);
523     return $encryptedTwice;
524 }
525
526 function dummyEncrypt($data)
527 {
528     return [
529         'key'  => base64_encode('0123456789abcdef') . "\n",
530         'iv'   => 't3jir1LHpICunvhlM76edQ==' . "\n",//random bytes
531         'blob' => base64_encode(
532             json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES)
533         ) . "\n",
534     ];
535 }
536
537 function addChunkedDiscoverRows(&$data, $games, $title)
538 {
539     $chunks = array_chunk($games, 4);
540     $first = true;
541     foreach ($chunks as $chunk) {
542         addDiscoverRow(
543             $data, $first ? $title : '',
544             $chunk
545         );
546         $first = false;
547     }
548 }
549
550 function addDiscoverRow(&$data, $title, $games)
551 {
552     $row = [
553         'title'     => $title,
554         'showPrice' => true,
555         'ranked'    => false,
556         'tiles'     => [],
557     ];
558     foreach ($games as $game) {
559         if (is_string($game)) {
560             //category link
561             $tilePos = count($data['tiles']);
562             $data['tiles'][$tilePos] = buildDiscoverCategoryTile($game);
563
564         } else {
565             //game
566             if (isset($game->links->original)) {
567                 //do not link unlocked games.
568                 // people an access them via the original games
569                 continue;
570             }
571             $tilePos = findTile($data['tiles'], $game->packageName);
572             if ($tilePos === null) {
573                 $tilePos = count($data['tiles']);
574                 $data['tiles'][$tilePos] = buildDiscoverGameTile($game);
575             }
576         }
577         $row['tiles'][] = $tilePos;
578     }
579     $data['rows'][] = $row;
580 }
581
582 function findTile($tiles, $packageName)
583 {
584     foreach ($tiles as $pos => $tile) {
585         if ($tile['package'] == $packageName) {
586             return $pos;
587         }
588     }
589     return null;
590 }
591
592 function buildDiscoverCategoryTile($title)
593 {
594     return [
595         'url'   => 'ouya://launcher/discover/' . categoryPath($title),
596         'image' => '',
597         'title' => $title,
598         'type'  => 'discover'
599     ];
600 }
601
602 function buildDiscoverGameTile($game)
603 {
604     $latestRelease = $game->latestRelease;
605     return [
606         'gamerNumbers' => $game->players,
607         'genres' => $game->genres,
608         'url' => 'ouya://launcher/details?app=' . $game->packageName,
609         'latestVersion' => [
610             'apk' => [
611                 'md5sum' => $latestRelease->md5sum,
612             ],
613             'versionNumber' => $latestRelease->name,
614             'uuid' => $latestRelease->uuid,
615         ],
616         'inAppPurchases' => $game->inAppPurchases,
617         'promotedProduct' => null,
618         'premium' => $game->premium,
619         'type' => 'app',
620         'package' => $game->packageName,
621         'updated_at' => strtotime($latestRelease->date),
622         'updatedAt' => $latestRelease->date,
623         'title' => $game->title,
624         'image' => $game->discover,
625         'contentRating' => $game->contentRating,
626         'rating' => [
627             'count' => $game->rating->count,
628             'average' => $game->rating->average,
629         ],
630         'promotedProduct' => buildProduct(getPromotedProduct($game)),
631     ];
632 }
633
634 function categoryPath($title)
635 {
636     return str_replace(['/', '\\', ' ', '+'], '_', $title);
637 }
638
639 function getAllAges($games)
640 {
641     $ages = [];
642     foreach ($games as $game) {
643         $ages[] = $game->contentRating;
644     }
645     return array_unique($ages);
646 }
647
648 function getAllGenres($games)
649 {
650     $genres = [];
651     foreach ($games as $game) {
652         $genres = array_merge($genres, $game->genres);
653     }
654     return array_unique($genres);
655 }
656
657 function addMissingGameProperties($game)
658 {
659     if (!isset($game->overview)) {
660         $game->overview = null;
661     }
662     if (!isset($game->description)) {
663         $game->description = '';
664     }
665     if (!isset($game->players)) {
666         $game->players = [1];
667     }
668     if (!isset($game->genres)) {
669         $game->genres = ['Unsorted'];
670     }
671     if (!isset($game->website)) {
672         $game->website = null;
673     }
674     if (!isset($game->contentRating)) {
675         $game->contentRating = 'Everyone';
676     }
677     if (!isset($game->premium)) {
678         $game->premium = false;
679     }
680     if (!isset($game->firstPublishedAt)) {
681         $game->firstPublishedAt = gmdate('c');
682     }
683
684     if (!isset($game->rating)) {
685         $game->rating = new stdClass();
686     }
687     if (!isset($game->rating->likeCount)) {
688         $game->rating->likeCount = 0;
689     }
690     if (!isset($game->rating->average)) {
691         $game->rating->average = 0;
692     }
693     if (!isset($game->rating->count)) {
694         $game->rating->count = 0;
695     }
696
697     $game->latestRelease = null;
698     $latestReleaseTimestamp = 0;
699     foreach ($game->releases as $release) {
700         if (!isset($release->publicSize)) {
701             $release->publicSize = 0;
702         }
703         if (!isset($release->nativeSize)) {
704             $release->nativeSize = 0;
705         }
706
707         $releaseTimestamp = strtotime($release->date);
708         if ($releaseTimestamp > $latestReleaseTimestamp) {
709             $game->latestRelease    = $release;
710             $latestReleaseTimestamp = $releaseTimestamp;
711         }
712     }
713     if ($game->latestRelease === null) {
714         error('No latest release for ' . $game->packageName);
715     }
716
717     if (!isset($game->media)) {
718         $game->media = [];
719     }
720
721     if (!isset($game->developer->uuid)) {
722         $game->developer->uuid = null;
723     }
724     if (!isset($game->developer->name)) {
725         $game->developer->name = 'unknown';
726     }
727     if (!isset($game->developer->supportEmail)) {
728         $game->developer->supportEmail = null;
729     }
730     if (!isset($game->developer->supportPhone)) {
731         $game->developer->supportPhone = null;
732     }
733     if (!isset($game->developer->founder)) {
734         $game->developer->founder = false;
735     }
736 }
737
738 /**
739  * Implements a sensible ranking system described in
740  * https://stackoverflow.com/a/1411268/2826013
741  */
742 function calculateRank(array $games)
743 {
744     $averageRatings = array_map(
745         function ($game) {
746             return $game->rating->average;
747         },
748         $games
749     );
750     $average = array_sum($averageRatings) / count($averageRatings);
751     $C = $average;
752     $m = 500;
753
754     foreach ($games as $game) {
755         $R = $game->rating->average;
756         $v = $game->rating->count;
757         $game->rating->rank = ($R * $v + $C * $m) / ($v + $m);
758     }
759 }
760
761 function getFirstVideoUrl($media)
762 {
763     foreach ($media as $medium) {
764         if ($medium->type == 'video') {
765             return $medium->url;
766         }
767     }
768     return null;
769 }
770
771 function getAllImageUrls($media)
772 {
773     $imageUrls = [];
774     foreach ($media as $medium) {
775         if ($medium->type == 'image') {
776             $imageUrls[] = $medium->url;
777         }
778     }
779     return $imageUrls;
780 }
781
782 function getPromotedProduct($game)
783 {
784     if (!isset($game->products) || !count($game->products)) {
785         return null;
786     }
787     foreach ($game->products as $gameProd) {
788         if ($gameProd->promoted) {
789             return $gameProd;
790         }
791     }
792     return null;
793 }
794
795 function removeMakeGames(array $games)
796 {
797     return filterByGenre($games, 'Tutorials', true);
798 }
799
800 function removeMakeGenres($genres)
801 {
802     $filtered = [];
803     foreach ($genres as $genre) {
804         if ($genre != 'Tutorials' && $genre != 'Builds') {
805             $filtered[] = $genre;
806         }
807     }
808     return $filtered;
809 }
810
811 function writeJson($path, $data)
812 {
813     global $wwwDir;
814     $fullPath = $wwwDir . $path;
815     $dir = dirname($fullPath);
816     if (!is_dir($dir)) {
817         mkdir($dir, 0777, true);
818     }
819     file_put_contents(
820         $fullPath,
821         json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n"
822     );
823 }
824
825 function error($msg)
826 {
827     fwrite(STDERR, $msg . "\n");
828     exit(1);
829 }
830 ?>