From: Christian Weiske <cweiske@cweiske.de>
Date: Mon, 4 Nov 2019 20:25:33 +0000 (+0100)
Subject: scripts to generate game files and internet archive import
X-Git-Url: https://git.cweiske.de/ouya-game-data.git/commitdiff_plain/ef5020152268a7503a1aa83b56b3fd0181067290

scripts to generate game files and internet archive import
---

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..5e39c23
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+old-data
diff --git a/bin/convert-original.php b/bin/convert-original.php
new file mode 100755
index 0000000..762e1ce
--- /dev/null
+++ b/bin/convert-original.php
@@ -0,0 +1,180 @@
+#!/usr/bin/env php
+<?php
+/**
+ * Convert OUYA storefront data to a game json file
+ *
+ * @author Christian Weiske <cweiske@cweiske.de>
+ */
+if (isset($argv[1]) && in_array($argv[1], ['-h', '--help'])) {
+    error(
+        'Usage: convert-original.php ouya-game-details.json ouya-game-download.json'
+    );
+}
+
+//cli arguments
+if (!isset($argv[1])) {
+    error('details json file parameter missing');
+}
+$detailsFile = $argv[1];
+
+if (!isset($argv[2])) {
+    error('download json file parameter missing');
+}
+$downloadFile = $argv[2];
+
+
+//file loading
+$detailsJson = file_get_contents($detailsFile);
+if ($detailsJson === false || trim($detailsJson) === '') {
+    error('Details file is empty');
+}
+$detailsData = json_decode($detailsJson);
+if ($detailsData === null) {
+    error('Details JSON cannot de loaded');
+}
+
+$downloadJson = file_get_contents($downloadFile);
+if ($downloadJson === false || trim($downloadJson) === '') {
+    error('Download file is empty');
+}
+$downloadData = json_decode($downloadJson);
+if ($downloadData === null) {
+    error('Download JSON cannot de loaded');
+}
+
+$package = basename($detailsFile, '.json');
+//data building
+$gameData = [
+    'uuid'        => $detailsData->app->uuid,
+    'package'     => $package,
+    'title'       => $detailsData->app->title,
+    'description' => $detailsData->app->description,
+    'players'     => $detailsData->app->gamerNumbers,
+    'genres'      => $detailsData->app->genres,
+    
+    'releases' => [
+        [
+            'name'       => $detailsData->app->versionNumber,
+            'uuid'       => $detailsData->app->latestVersion,
+            'date'       => $detailsData->app->publishedAt,
+            'url'        => $downloadData->app->downloadLink,
+            'size'       => $downloadData->app->fileSize,
+            'md5sum'     => $detailsData->app->md5sum,
+            'publicSize' => $detailsData->app->publicSize,
+            'nativeSize' => $detailsData->app->nativeSize,
+        ]
+    ],
+
+    'media' => [
+        'discover'    => 'http://ouya.cweiske.de/game-images/' . strtolower($package) . '/discover',
+        'large'       => newImage($detailsData->app->mainImageFullUrl),
+        'video'       => $detailsData->app->videoUrl,
+        'screenshots' => newImages($detailsData->app->filepickerScreenshots),
+    ],
+
+    'developer' => [
+        'name'         => $detailsData->app->developer,
+        'supportEmail' => $detailsData->app->supportEmailAddress,
+        'supportPhone' => $detailsData->app->supportPhone,
+        'founder'      => $detailsData->app->founder,
+    ],
+
+    'contentRating'    => $detailsData->app->contentRating,
+    'website'          => $detailsData->app->website,
+    'firstPublishedAt' => $detailsData->app->firstPublishedAt,
+    'inAppPurchases'   => false,//FIXME: we would need discover data here
+    'overview'         => $detailsData->app->overview,
+    'premium'          => $detailsData->app->premium,
+
+    'rating' => [
+        'likeCount' => $detailsData->app->likeCount,
+        'average'   => $detailsData->app->ratingAverage,
+        'count'     => $detailsData->app->ratingCount,
+    ],
+];
+
+if (isset($detailsData->app->promotedProduct)) {
+    $gameData['products'][] = [
+        'promoted'      => true,
+        'identifier'    => $detailsData->app->promotedProduct->identifier,
+        'name'          => $detailsData->app->promotedProduct->name,
+        'description'   => $detailsData->app->promotedProduct->description,
+        'localPrice'    => $detailsData->app->promotedProduct->localPrice,
+        'originalPrice' => $detailsData->app->promotedProduct->originalPrice,
+        'currency'      => $detailsData->app->promotedProduct->currency,
+    ];
+}
+
+$iaDataFiles = glob(__DIR__ . '/../old-data/ia-data/ouya_' . $package . '_*.json');
+if (count($iaDataFiles)) {
+    $iaPkg = [];
+    foreach ($iaDataFiles as $iaJsonFile) {
+        $iaData = json_decode(file_get_contents($iaJsonFile));
+        foreach ($iaData->files as $iaFile) {
+            if ($iaFile->format == 'Android Package Archive') {
+                $iaSlug = basename($iaJsonFile, '.json');
+                $iaFile->url = 'https://archive.org/download/' . $iaSlug . '/' . $iaFile->name;
+                $versionName = explode('_', $iaSlug)[2];
+                $iaPkg[$versionName] = $iaFile;
+            }
+        }
+    }
+
+    //update existing release
+    $exVersion = $gameData['releases'][0]['name'];
+    if (isset($iaPkg[$exVersion])) {
+        $gameData['releases'][0]['url'] = $iaPkg[$exVersion]->url;
+        unset($iaPkg[$exVersion]);
+    }
+    foreach ($iaPkg as $iaVersion => $iaApk) {
+        $gameData['releases'][] = [
+            'name'       => $iaVersion,
+            'uuid'       => null,
+            'date'       => '2010-01-01T00:00:00Z',//gmdate('c', $iaApk->mtime),
+            'url'        => $iaApk->url,
+            'size'       => $iaApk->size,
+            'md5sum'     => $iaApk->md5,
+            'publicSize' => 0,//FIXME
+            'nativeSize' => 0,//FIXME
+        ];
+    }
+    var_dump($iaPkg, $gameData);die();
+}
+
+
+echo json_encode($gameData, JSON_PRETTY_PRINT) . "\n";
+
+function newImage($imageUrl)
+{
+    return $imageUrl;
+    static $mapping;
+    if ($mapping === null) {
+        $hdl = fopen(__DIR__ . '/../old-data/map-game-images.csv', 'r');
+        if (!$hdl) {
+            error('Cannot load image url map file');
+        }
+        $mapping = [];
+        while ($data = fgetcsv($hdl, 4096, ',')) {
+            if (count($data) == 2) {
+                $mapping[$data[0]] = $data[1];
+            }
+        }
+    }
+    return $mapping[$imageUrl] ?? $imageUrl;
+}
+
+function newImages($urls)
+{
+    $new = [];
+    foreach ($urls as $url) {
+        $new[] = newImage($url);
+    }
+    return $new;
+}
+
+function error($msg)
+{
+    file_put_contents('php://stderr', $msg . "\n");
+    exit(1);
+}
+?>
diff --git a/bin/copy-game-images.php b/bin/copy-game-images.php
new file mode 100644
index 0000000..d8ac102
--- /dev/null
+++ b/bin/copy-game-images.php
@@ -0,0 +1,55 @@
+<?php
+$gamefiles = glob('devs.ouya.tv/api/v1/apps/*.json');
+
+foreach ($gamefiles as $file) {
+    echo "Processing $file\n";
+    $data = json_decode(file_get_contents($file));
+    if ($data === null) {
+        echo "error opening " . $file . "\n";
+        exit(1);
+    }
+    $package = strtolower(basename($file, '.json'));
+
+    $dir = 'game-images/' . $package . '/';
+    if (!is_dir($dir)) {
+        mkdir($dir, 0777, true);
+    }
+
+    $pos = 0;
+    foreach ($data->app->filepickerScreenshots as $imageUrl) {
+        $pos++;
+        copyImageUrl($imageUrl, $dir . 'screenshot-' . $pos);
+    }
+
+    copyImageUrl($data->app->mainImageFullUrl, $dir . 'main');
+    echo "\n";
+}
+
+function copyImageUrl($imageUrl, $newPath)
+{
+    $imageLocal = getLocalPath($imageUrl);
+    if (!file_exists($imageLocal)) {
+        echo "Local file not found: $imageLocal\n";
+        return;
+    }
+    if (file_exists($newPath)) {
+        echo "S";
+        return;
+    }
+    
+    echo ".";
+    copy($imageLocal, $newPath);
+    file_put_contents(
+        'map-game-images.csv',
+        $imageUrl . ','
+        . 'http://ouya.cweiske.de/' . $newPath
+        . "\n",
+        FILE_APPEND
+    );
+}
+
+function getLocalPath($imageUrl)
+{
+    return str_replace('https://', '', $imageUrl);
+}
+?>
diff --git a/bin/download-discover-images-ia.php b/bin/download-discover-images-ia.php
new file mode 100755
index 0000000..6bd50dc
--- /dev/null
+++ b/bin/download-discover-images-ia.php
@@ -0,0 +1,35 @@
+#!/usr/bin/env php
+<?php
+$files = glob(__DIR__ . '/../old-data/ia-data/*.json');
+foreach ($files as $path) {
+    $file = basename($path);
+    $parts = explode('_', $file);
+    if (count($parts) != 3) {
+        continue;
+    }
+    if ($parts[0] != 'ouya') {
+        continue;
+    }
+    $package = $parts[1];
+    echo $package . "\n";
+
+    $data = json_decode(file_get_contents($path));
+    foreach ($data->files as $dfile) {
+        if ($dfile->name == '00ICON.png') {
+            $url = 'http://' . $data->d1
+                 . $data->dir . '/' . $dfile->name;
+            $dlfile = '/media/cweiske/videos/ouya-backup/game-images/' . strtolower($package) . '/discover';
+            if (file_exists($dlfile)) {
+                echo " S\n";
+                break;
+            }
+            echo ' ' . $url . "\n";
+            $cmd = 'curl -s ' . escapeshellarg($url)
+                 . ' -o ' . escapeshellarg($dlfile);
+            passthru($cmd);
+            break;
+        }
+    }
+    die();
+}
+?>
diff --git a/bin/fetch-ia-data.php b/bin/fetch-ia-data.php
new file mode 100644
index 0000000..beb0ff8
--- /dev/null
+++ b/bin/fetch-ia-data.php
@@ -0,0 +1,15 @@
+#!/usr/bin/env php
+<?php
+chdir(__DIR__ . '/../old-data/');
+
+// see ouya-games.rst to update that one
+$packages = file('ia-packages');
+
+foreach ($packages as $package) {
+    $package = trim($package);
+    $url = 'https://archive.org/metadata/' . $package;
+    $cmd = 'curl -s ' . escapeshellarg($url)
+         . ' | jq . > ia-data/' . $package . '.json';
+    passthru($cmd);
+}
+?>
diff --git a/bin/find-game-images-ia.php b/bin/find-game-images-ia.php
new file mode 100755
index 0000000..c41cf8d
--- /dev/null
+++ b/bin/find-game-images-ia.php
@@ -0,0 +1,77 @@
+#!/usr/bin/env php
+<?php
+chdir(__DIR__ . '/../old-data/');
+$gamefiles = glob('devs.ouya.tv/api/v1/apps/*.json');
+//$gamefiles = glob('devs.ouya.tv/api/v1/apps/net.froem*.json');
+
+foreach ($gamefiles as $file) {
+    echo "Processing $file\n";
+    $data = json_decode(file_get_contents($file));
+    if ($data === null) {
+        echo "error opening " . $file . "\n";
+        exit(1);
+    }
+    $package = basename($file, '.json');
+
+    $iaImages = loadIaImages($package);
+
+    $pos = 0;
+    foreach ($data->app->filepickerScreenshots as $imageUrl) {
+        $pos++;
+        mapIaImage($iaImages, $imageUrl);
+    }
+
+    mapIaImage($iaImages, $data->app->mainImageFullUrl);
+    //die();
+}
+
+function mapIaImage($iaImages, $imageUrl)
+{
+    $newUrl = findIaImage($iaImages, $imageUrl);
+    if ($newUrl !== $imageUrl) {
+        echo " ok\n";
+        file_put_contents(
+            'map-game-images.ia.csv',
+            $imageUrl . ',' . $newUrl . "\n",
+            FILE_APPEND
+        );
+        return;
+    }
+    //not in internet archive
+    echo " Missing in IA: $imageUrl\n";
+}
+
+function findIaImage($iaImages, $imageUrl)
+{
+    // https://d3e4aumcqn8cw3.cloudfront.net/api/file/tC4RIGJLQvG2uG1av9jN
+    $imageName = basename($imageUrl);
+    if (isset($iaImages[$imageName . '.png'])) {
+        return $iaImages[$imageName . '.png'];
+    }
+    if (isset($iaImages[$imageName . '.jpg'])) {
+        return $iaImages[$imageName . '.jpg'];
+    }
+    return $imageUrl;
+}
+
+function loadIaImages($package)
+{
+    $images = [];
+    $iaDataFiles = glob('ia-data/ouya_' . $package . '_*.json');
+    foreach ($iaDataFiles as $iaJsonFile) {
+        $iaSlug = basename($iaJsonFile, '.json');
+        $data = json_decode(file_get_contents($iaJsonFile));
+        foreach ($data->files as $file) {
+            if ($file->source != 'original') {
+                continue;
+            }
+            if ($file->format != 'JPEG' && $file->format != 'PNG') {
+                continue;
+            }
+
+            $images[$file->name] = 'https://archive.org/download/' . $iaSlug . '/' . $file->name;
+        }
+    }
+    return $images;
+}
+?>
diff --git a/bin/find-ia-missing.php b/bin/find-ia-missing.php
new file mode 100755
index 0000000..2131ee2
--- /dev/null
+++ b/bin/find-ia-missing.php
@@ -0,0 +1,106 @@
+#!/usr/bin/env php
+<?php
+/**
+ * Find ouya game images missing in the internet archive
+ */
+chdir(__DIR__ . '/../old-data/');
+
+$iaDataFiles = glob('ia-data/ouya_*_*.json');
+foreach ($iaDataFiles as $iaJsonFile) {
+    $iaPackage = basename($iaJsonFile, '.json');
+    $parts = explode('_', $iaPackage);
+    if (count($parts) != 3) {
+        continue;
+    }
+    $package = $parts[1];
+    fwrite(STDERR, $package . "\n");
+
+    $iaImages = loadIaImages($iaJsonFile);
+
+    $ouyaDetailsFile = 'devs.ouya.tv/api/v1/apps/' . $package . '.json';
+    if (!file_exists($ouyaDetailsFile)) {
+        fwrite(STDERR, " ERR: ouya file missing $ouyaDetailsFile\n");
+        continue;
+    }
+    $details = json_decode(file_get_contents($ouyaDetailsFile));
+    if ($details === null) {
+        fwrite(STDERR, "error opening " . $ouyaDetailsFile . "\n");
+        exit(1);
+    }
+
+    $pos = 0;
+    foreach ($details->app->filepickerScreenshots as $imageUrl) {
+        $pos++;
+        findIaImage($iaImages, $imageUrl, $iaPackage);
+    }
+
+    findIaImage($iaImages, $details->app->mainImageFullUrl, $iaPackage);
+}
+
+
+
+function findIaImage($iaImages, $imageUrl, $iaPackage)
+{
+    // https://d3e4aumcqn8cw3.cloudfront.net/api/file/tC4RIGJLQvG2uG1av9jN
+    $imageName = basename($imageUrl);
+    if (isset($iaImages[$imageName . '.png'])) {
+        return;
+    }
+    if (isset($iaImages[$imageName . '.jpg'])) {
+        return;
+    }
+
+    $cwUrl = str_replace('https://', 'http://tmp.cweiske.de/', $imageUrl);
+
+    $localPath = str_replace('https://', '/media/cweiske/videos/ouya-backup/', $imageUrl);
+    if (!file_exists($localPath)) {
+        fwrite(STDERR, " local file not found: $localPath\n");
+    }
+
+    echo $iaPackage . ',' . $cwUrl . ',' . $imageUrl . "\n";
+}
+
+function loadIaImages($iaJsonFile)
+{
+    $iaSlug = basename($iaJsonFile, '.json');
+    $data = json_decode(file_get_contents($iaJsonFile));
+    $images = [];
+    foreach ($data->files as $file) {
+        if ($file->source != 'original') {
+            continue;
+        }
+        if ($file->format != 'JPEG' && $file->format != 'PNG') {
+            continue;
+        }
+
+        $images[$file->name] = 'https://archive.org/download/' . $iaSlug . '/' . $file->name;
+    }
+    return $images;
+}
+
+exit(0);
+
+$gamefiles = glob('devs.ouya.tv/api/v1/apps/*.json');
+//$gamefiles = glob('devs.ouya.tv/api/v1/apps/net.froem*.json');
+
+foreach ($gamefiles as $file) {
+    echo "Processing $file\n";
+    //die();
+}
+
+function mapIaImage($iaImages, $imageUrl)
+{
+    $newUrl = findIaImage($iaImages, $imageUrl);
+    if ($newUrl !== $imageUrl) {
+        echo " ok\n";
+        file_put_contents(
+            'map-game-images.ia.csv',
+            $imageUrl . ',' . $newUrl . "\n",
+            FILE_APPEND
+        );
+        return;
+    }
+    //not in internet archive
+    echo " Missing in IA: $imageUrl\n";
+}
+?>
diff --git a/ouya-games.rst b/ouya-games.rst
new file mode 100644
index 0000000..08fd4a1
--- /dev/null
+++ b/ouya-games.rst
@@ -0,0 +1,19 @@
+**********************************
+OUYA games in the Internet Archive
+**********************************
+
+https://archive.org/details/ouyalibrary
+
+Fetch all items in the collection::
+
+  $ ia search collection:ouyalibrary | jq -r .identifier > ia-packages
+
+Fetch data/files for one item::
+
+  $ curl -s https://archive.org/metadata/ouya_de.eiswuxe.blookid2_1.6|jq .
+
+Extract discover images::
+
+  $ cat discover.json | jq 'reduce (.tiles[]| select(.package!=null)) as $i ({}; .[$i.package] = $i.image)'
+
+(We simply use a scaled version of the main image as discover image)