feat: port the snow report import (ep:import-snowreport)
Replaces the TYPO3 10 scheduler task, which stopped with PHP 7.4. Product pages rendered by TYPO3 13 show the snow report of their region, so its data has to be kept current here. Plain DBAL instead of Extbase persistence; records are matched by vendor_uid and updated in place, so the regions keep their snow report. Importing the feed TYPO3 10 imported last leaves every column unchanged. The feed is kept in var/transient (shared) instead of fileadmin, which still belongs to the TYPO3 10 installation. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
frommeandClaude Opus 5.5 committed
1 parent
42e02095ce
commit
4d4baa1124
2 files changed
+369
No files matched your search
@@ -0,0 +1,38 @@
|
||||
<?php
|
||||
|
||||
namespace EP\EpProducts\Command;
|
||||
|
||||
use EP\EpProducts\Service\SnowreportImportService;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
#[AsCommand(name: 'ep:import-snowreport', description: 'Downloads and imports the snow report feed (skiresort-service.com)')]
|
||||
class SnowreportCommand extends Command
|
||||
{
|
||||
public function __construct(
|
||||
private readonly SnowreportImportService $snowreportImportService,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this->addOption('file', null, InputOption::VALUE_REQUIRED, 'Import this feed file instead of downloading');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
$file = $input->getOption('file');
|
||||
if (null === $file && !$this->snowreportImportService->downloadFeed()) {
|
||||
$output->writeln('<comment>Download failed, importing the previous feed (see the TYPO3 log).</comment>');
|
||||
}
|
||||
|
||||
$count = $this->snowreportImportService->import($file);
|
||||
$output->writeln(sprintf('%d snow reports imported.', $count));
|
||||
|
||||
return $count > 0 ? Command::SUCCESS : Command::FAILURE;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
<?php
|
||||
|
||||
namespace EP\EpProducts\Service;
|
||||
|
||||
use Psr\Log\LoggerAwareInterface;
|
||||
use Psr\Log\LoggerAwareTrait;
|
||||
use TYPO3\CMS\Core\Cache\CacheManager;
|
||||
use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
|
||||
use TYPO3\CMS\Core\Core\Environment;
|
||||
use TYPO3\CMS\Core\Database\Connection;
|
||||
use TYPO3\CMS\Core\Database\ConnectionPool;
|
||||
use TYPO3\CMS\Core\Database\Query\Restriction\DeletedRestriction;
|
||||
use TYPO3\CMS\Core\Http\RequestFactory;
|
||||
|
||||
/**
|
||||
* Imports the snow report feed of skiresort-service.com (snow levels, lifts, webcams per ski area)
|
||||
*
|
||||
* Port of the TYPO3 10 import with plain DBAL instead of Extbase persistence. Records are matched by
|
||||
* vendor_uid and updated in place, so the regions keep pointing at their snow report; records no
|
||||
* longer in the feed are flagged deleted. The feed only answers the production server.
|
||||
*/
|
||||
class SnowreportImportService implements LoggerAwareInterface
|
||||
{
|
||||
use LoggerAwareTrait;
|
||||
|
||||
private const FEED_URL = 'https://www.skiresort-service.com/xml-feed/';
|
||||
|
||||
// A healthy response arrives in well under a second; an outage should fail visibly
|
||||
private const HTTP_CONNECT_TIMEOUT = 5;
|
||||
private const HTTP_TIMEOUT = 30;
|
||||
|
||||
private const SNOWREPORT_TABLE = 'tx_epproducts_domain_model_snowreport';
|
||||
private const WEBCAM_TABLE = 'tx_epproducts_domain_model_webcam';
|
||||
|
||||
private const TYPE_INT = 'int';
|
||||
private const TYPE_STRING = 'string';
|
||||
private const TYPE_BOOL = 'bool';
|
||||
private const TYPE_DATE = 'date';
|
||||
|
||||
/**
|
||||
* XML element => column and type. XML has no types, so each field is declared here and converted
|
||||
* strictly; unexpected values are logged and fall back to the type's empty value.
|
||||
*/
|
||||
private const MAP = [
|
||||
'region_id' => ['column' => 'vendor_uid', 'type' => self::TYPE_INT],
|
||||
'region_name' => ['column' => 'region_name', 'type' => self::TYPE_STRING],
|
||||
'region_offen' => ['column' => 'opened', 'type' => self::TYPE_BOOL],
|
||||
'gletschergebiet' => ['column' => 'glacier', 'type' => self::TYPE_BOOL],
|
||||
'schneehoehe_berg' => ['column' => 'snow_level_mountain', 'type' => self::TYPE_INT],
|
||||
'schneehoehe_tal' => ['column' => 'snow_level_valley', 'type' => self::TYPE_INT],
|
||||
'schneequalitaet' => ['column' => 'snow_quality', 'type' => self::TYPE_STRING],
|
||||
'letzter_schneefall' => ['column' => 'last_snowfall', 'type' => self::TYPE_DATE],
|
||||
'lifte_gesamt' => ['column' => 'lifts_total', 'type' => self::TYPE_INT],
|
||||
'offene_lifte' => ['column' => 'lifts_opened', 'type' => self::TYPE_INT],
|
||||
'letzte_aktualisierung' => ['column' => 'last_update', 'type' => self::TYPE_DATE],
|
||||
'hoehemax' => ['column' => 'height_max', 'type' => self::TYPE_INT],
|
||||
'hoehemin' => ['column' => 'height_min', 'type' => self::TYPE_INT],
|
||||
'datum_saisonstart' => ['column' => 'season_start', 'type' => self::TYPE_DATE],
|
||||
'datum_saisonende' => ['column' => 'season_end', 'type' => self::TYPE_DATE],
|
||||
];
|
||||
|
||||
private const BOOLEANS = [
|
||||
'true' => true, '1' => true, 'ja' => true, 'yes' => true,
|
||||
'false' => false, '0' => false, 'nein' => false, 'no' => false,
|
||||
];
|
||||
|
||||
/** @var array<string, true> distinct conversion problems of the current import */
|
||||
private array $conversionWarnings = [];
|
||||
|
||||
public function __construct(
|
||||
private readonly ConnectionPool $connectionPool,
|
||||
private readonly RequestFactory $requestFactory,
|
||||
private readonly ExtensionConfiguration $extensionConfiguration,
|
||||
private readonly CacheManager $cacheManager,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Last downloaded feed; kept in the shared var/transient so a failed download imports the previous one
|
||||
*/
|
||||
public function getFeedPath(): string
|
||||
{
|
||||
return Environment::getVarPath() . '/transient/ep_products_import/snowreport.xml';
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces the local feed only with a usable XML document; any failure keeps the existing file
|
||||
*/
|
||||
public function downloadFeed(): bool
|
||||
{
|
||||
$path = $this->getFeedPath();
|
||||
|
||||
try {
|
||||
$response = $this->requestFactory->request(self::FEED_URL, 'GET', [
|
||||
'connect_timeout' => self::HTTP_CONNECT_TIMEOUT,
|
||||
'timeout' => self::HTTP_TIMEOUT,
|
||||
'headers' => ['Accept' => 'application/xml, text/xml'],
|
||||
'http_errors' => false,
|
||||
]);
|
||||
if (200 !== $response->getStatusCode()) {
|
||||
throw new \RuntimeException(sprintf('unexpected status code %d', $response->getStatusCode()), 1758000001);
|
||||
}
|
||||
$xmlContent = (string)$response->getBody();
|
||||
$this->parseFeed($xmlContent);
|
||||
} catch (\Throwable $e) {
|
||||
$this->logger?->error(sprintf('Snowreport feed download failed (%s), keeping the existing file.', $e->getMessage()));
|
||||
return false;
|
||||
}
|
||||
|
||||
// Written next to the target and moved into place: the file is either the old or the new document
|
||||
if (!is_dir(dirname($path))) {
|
||||
mkdir(dirname($path), 0775, true);
|
||||
}
|
||||
$temporaryPath = $path . '.tmp';
|
||||
if (false === file_put_contents($temporaryPath, $xmlContent) || !rename($temporaryPath, $path)) {
|
||||
@unlink($temporaryPath);
|
||||
$this->logger?->error(sprintf('Snowreport feed could not be written to "%s".', $path));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return int number of snow reports imported
|
||||
*/
|
||||
public function import(?string $file = null): int
|
||||
{
|
||||
$file ??= $this->getFeedPath();
|
||||
if (!is_file($file)) {
|
||||
$this->logger?->error(sprintf('Snowreport import failed: file "%s" does not exist.', $file));
|
||||
return 0;
|
||||
}
|
||||
|
||||
try {
|
||||
$xmlData = $this->parseFeed((string)file_get_contents($file));
|
||||
} catch (\RuntimeException $e) {
|
||||
$this->logger?->error(sprintf('Snowreport import failed: %s', $e->getMessage()));
|
||||
return 0;
|
||||
}
|
||||
|
||||
$this->conversionWarnings = [];
|
||||
$pid = (int)($this->extensionConfiguration->get('ep_products')['importStoragePid'] ?? 0);
|
||||
$now = time();
|
||||
$snowreports = $this->loadIndex(self::SNOWREPORT_TABLE);
|
||||
$webcams = $this->loadIndex(self::WEBCAM_TABLE);
|
||||
$validSnowreports = [];
|
||||
$validWebcams = [];
|
||||
$snowreportConnection = $this->connectionPool->getConnectionForTable(self::SNOWREPORT_TABLE);
|
||||
$webcamConnection = $this->connectionPool->getConnectionForTable(self::WEBCAM_TABLE);
|
||||
|
||||
$count = 0;
|
||||
foreach ($xmlData->schneemeldung as $reportXml) {
|
||||
$row = [];
|
||||
foreach (self::MAP as $key => $field) {
|
||||
if (!isset($reportXml->{$key})) {
|
||||
$this->conversionWarnings[sprintf('Element <%s> missing in feed.', $key)] = true;
|
||||
$row[$field['column']] = $this->getEmptyValue($field['type']);
|
||||
continue;
|
||||
}
|
||||
$row[$field['column']] = $this->convertValue((string)$reportXml->{$key}, $field['type'], $key);
|
||||
}
|
||||
$vendorUid = (int)$row['vendor_uid'];
|
||||
$row = array_map(fn ($value) => is_bool($value) ? (int)$value : $value, $row);
|
||||
$row['tstamp'] = $now;
|
||||
|
||||
if (isset($snowreports[$vendorUid])) {
|
||||
$snowreportUid = $snowreports[$vendorUid];
|
||||
$snowreportConnection->update(self::SNOWREPORT_TABLE, $row, ['uid' => $snowreportUid]);
|
||||
} else {
|
||||
$snowreportConnection->insert(self::SNOWREPORT_TABLE, $row + ['pid' => $pid, 'crdate' => $now]);
|
||||
$snowreportUid = (int)$snowreportConnection->lastInsertId();
|
||||
$snowreports[$vendorUid] = $snowreportUid;
|
||||
}
|
||||
$validSnowreports[] = $vendorUid;
|
||||
|
||||
foreach ($reportXml->webcam ? $reportXml->webcam->children() : [] as $item) {
|
||||
$include = $this->convertValue((string)$item->aktuell, self::TYPE_BOOL, 'aktuell');
|
||||
$available = $this->convertValue((string)$item->verf, self::TYPE_BOOL, 'verf');
|
||||
$webcamVendorUid = (string)(int)$item->uid;
|
||||
|
||||
// Cams that are neither current nor already known are skipped
|
||||
if (!isset($webcams[$webcamVendorUid]) && !$include) {
|
||||
continue;
|
||||
}
|
||||
$webcamRow = [
|
||||
'url' => (string)$item->url,
|
||||
'name' => (string)$item->dt,
|
||||
'available' => (int)($available && $include),
|
||||
'tstamp' => $now,
|
||||
];
|
||||
if (isset($webcams[$webcamVendorUid])) {
|
||||
// Known cams stay with the snow report they were first imported for, as in TYPO3 10
|
||||
$webcamConnection->update(self::WEBCAM_TABLE, $webcamRow, ['uid' => $webcams[$webcamVendorUid]]);
|
||||
} else {
|
||||
$webcamConnection->insert(self::WEBCAM_TABLE, $webcamRow + [
|
||||
'pid' => $pid,
|
||||
'crdate' => $now,
|
||||
'vendor_uid' => $webcamVendorUid,
|
||||
'snowreport' => $snowreportUid,
|
||||
]);
|
||||
$webcams[$webcamVendorUid] = (int)$webcamConnection->lastInsertId();
|
||||
}
|
||||
$validWebcams[] = $webcamVendorUid;
|
||||
}
|
||||
|
||||
$count++;
|
||||
}
|
||||
|
||||
$this->flagObsolete(self::WEBCAM_TABLE, array_keys($webcams), $validWebcams, $now);
|
||||
$this->flagObsolete(self::SNOWREPORT_TABLE, array_keys($snowreports), $validSnowreports, $now);
|
||||
// Inline child counter, which the plain SQL above does not maintain
|
||||
$snowreportConnection->executeStatement(
|
||||
'UPDATE ' . self::SNOWREPORT_TABLE . ' s SET s.webcams = ('
|
||||
. ' SELECT COUNT(*) FROM ' . self::WEBCAM_TABLE . ' w WHERE w.snowreport = s.uid AND w.deleted = 0)'
|
||||
);
|
||||
|
||||
// Product pages show the snow report of their region
|
||||
$this->cacheManager->flushCachesInGroup('pages');
|
||||
|
||||
if ([] !== $this->conversionWarnings) {
|
||||
$messages = array_keys($this->conversionWarnings);
|
||||
$this->logger?->warning(sprintf('Snowreport import: %d unexpected value(s) in feed: %s', count($messages), implode(' | ', array_slice($messages, 0, 10))));
|
||||
}
|
||||
$this->logger?->info(sprintf('Snowreport import successful (%d records).', $count));
|
||||
|
||||
return $count;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rejects unusable documents: an empty or truncated feed would otherwise flag every record as deleted
|
||||
*/
|
||||
private function parseFeed(string $xmlContent): \SimpleXMLElement
|
||||
{
|
||||
if ('' === trim($xmlContent)) {
|
||||
throw new \RuntimeException('response is empty', 1758000002);
|
||||
}
|
||||
|
||||
$useInternalErrors = libxml_use_internal_errors(true);
|
||||
libxml_clear_errors();
|
||||
$xmlData = simplexml_load_string($xmlContent);
|
||||
$errors = libxml_get_errors();
|
||||
libxml_clear_errors();
|
||||
libxml_use_internal_errors($useInternalErrors);
|
||||
|
||||
if (false === $xmlData) {
|
||||
$message = count($errors) > 0 ? trim($errors[0]->message) : 'unknown error';
|
||||
throw new \RuntimeException(sprintf('response is not valid XML (%s)', $message), 1758000003);
|
||||
}
|
||||
if (0 === $xmlData->schneemeldung->count()) {
|
||||
throw new \RuntimeException('response contains no <schneemeldung> elements', 1758000004);
|
||||
}
|
||||
|
||||
return $xmlData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Not deleted records (hidden ones included: the import still manages them) as vendor_uid => uid
|
||||
*/
|
||||
private function loadIndex(string $table): array
|
||||
{
|
||||
$queryBuilder = $this->connectionPool->getQueryBuilderForTable($table);
|
||||
$queryBuilder->getRestrictions()->removeAll()->add(new DeletedRestriction());
|
||||
$index = [];
|
||||
foreach ($queryBuilder->select('uid', 'vendor_uid')->from($table)->orderBy('uid')->executeQuery()->fetchAllAssociative() as $row) {
|
||||
$index[(string)$row['vendor_uid']] ??= (int)$row['uid'];
|
||||
}
|
||||
|
||||
return $index;
|
||||
}
|
||||
|
||||
private function flagObsolete(string $table, array $currentVendorUids, array $validVendorUids, int $now): void
|
||||
{
|
||||
$obsolete = array_values(array_diff(array_map('strval', $currentVendorUids), array_map('strval', $validVendorUids)));
|
||||
if ([] === $obsolete) {
|
||||
return;
|
||||
}
|
||||
|
||||
$queryBuilder = $this->connectionPool->getQueryBuilderForTable($table);
|
||||
$queryBuilder
|
||||
->update($table)
|
||||
->set('deleted', 1)
|
||||
->set('tstamp', $now)
|
||||
->where($queryBuilder->expr()->in('vendor_uid', $queryBuilder->createNamedParameter($obsolete, Connection::PARAM_STR_ARRAY)))
|
||||
->executeStatement();
|
||||
}
|
||||
|
||||
private function convertValue(string $raw, string $type, string $key): int|string|bool
|
||||
{
|
||||
$raw = trim($raw);
|
||||
|
||||
// Empty elements mean "not available yet" (snow levels out of season), which is expected
|
||||
if ('' === $raw) {
|
||||
return $this->getEmptyValue($type);
|
||||
}
|
||||
|
||||
switch ($type) {
|
||||
case self::TYPE_STRING:
|
||||
return $raw;
|
||||
case self::TYPE_INT:
|
||||
if (!preg_match('/^-?\d+$/', $raw)) {
|
||||
$this->conversionWarnings[sprintf('<%s> expected an integer, got "%s".', $key, $raw)] = true;
|
||||
return 0;
|
||||
}
|
||||
return (int)$raw;
|
||||
case self::TYPE_BOOL:
|
||||
$normalized = strtolower($raw);
|
||||
if (!array_key_exists($normalized, self::BOOLEANS)) {
|
||||
$this->conversionWarnings[sprintf('<%s> expected a boolean, got "%s".', $key, $raw)] = true;
|
||||
return false;
|
||||
}
|
||||
return self::BOOLEANS[$normalized];
|
||||
case self::TYPE_DATE:
|
||||
if (!preg_match('/^\d{4}-\d{2}-\d{2}([ T]\d{2}:\d{2}(:\d{2})?)?$/', $raw)) {
|
||||
$this->conversionWarnings[sprintf('<%s> expected a date, got "%s".', $key, $raw)] = true;
|
||||
return 0;
|
||||
}
|
||||
return (new \DateTime($raw))->getTimestamp();
|
||||
}
|
||||
|
||||
throw new \InvalidArgumentException(sprintf('Unknown field type "%s" declared for <%s>.', $type, $key), 1758000000);
|
||||
}
|
||||
|
||||
private function getEmptyValue(string $type): int|string|bool
|
||||
{
|
||||
return match ($type) {
|
||||
self::TYPE_STRING => '',
|
||||
self::TYPE_BOOL => false,
|
||||
default => 0,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user