feat: api endpoints and xml sync for contingents data
This commit is contained in:
Björn Fromme
+221
-149
@@ -5,12 +5,11 @@ declare(strict_types=1);
|
||||
namespace App\Command;
|
||||
|
||||
use App\BusProNet\Model\XmlExportInfo;
|
||||
use App\BusProNet\XmlLoader\TravelLoader;
|
||||
use App\Service\TravelDataProvider;
|
||||
use App\Service\TravelSnapshotManager;
|
||||
use App\Model\BpnXmlSnapshotRefreshResult;
|
||||
use App\Model\BpnXmlSyncTarget;
|
||||
use App\Service\BpnXmlSyncManager;
|
||||
use App\Service\BpnXmlSnapshotRefreshManager;
|
||||
use League\Flysystem\FilesystemException;
|
||||
use League\Flysystem\FilesystemOperator;
|
||||
use League\Flysystem\StorageAttributes;
|
||||
use Psr\Log\LoggerInterface;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
@@ -19,7 +18,6 @@ use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use Symfony\Component\Console\Style\SymfonyStyle;
|
||||
use Symfony\Contracts\Cache\CacheInterface;
|
||||
|
||||
#[AsCommand(
|
||||
name: 'app:bpn:xml-sync',
|
||||
@@ -29,20 +27,10 @@ class BpnXmlSyncCommand extends Command
|
||||
{
|
||||
use LockableTrait;
|
||||
|
||||
private const CACHE_KEYS_TO_INVALIDATE = [
|
||||
'bpn_travels_mapping',
|
||||
'bpn_hotels',
|
||||
'bpn_pickups',
|
||||
];
|
||||
|
||||
public function __construct(
|
||||
private readonly FilesystemOperator $xmlSource,
|
||||
private readonly FilesystemOperator $xmlExport,
|
||||
private readonly CacheInterface $cache,
|
||||
private readonly BpnXmlSyncManager $syncManager,
|
||||
private readonly BpnXmlSnapshotRefreshManager $snapshotRefreshManager,
|
||||
private readonly LoggerInterface $logger,
|
||||
private readonly TravelLoader $travelLoader,
|
||||
private readonly TravelDataProvider $travelDataService,
|
||||
private readonly TravelSnapshotManager $snapshotService,
|
||||
) {
|
||||
parent::__construct();
|
||||
}
|
||||
@@ -65,177 +53,261 @@ class BpnXmlSyncCommand extends Command
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
$force = $input->getOption('force');
|
||||
$dryRun = $input->getOption('dry-run');
|
||||
return $this->sync(
|
||||
(bool) $input->getOption('force'),
|
||||
(bool) $input->getOption('dry-run'),
|
||||
$io,
|
||||
$output,
|
||||
);
|
||||
}
|
||||
|
||||
$remoteInfo = $this->readRemoteInfo();
|
||||
private function sync(bool $force, bool $dryRun, SymfonyStyle $io, OutputInterface $output): int
|
||||
{
|
||||
$totalSyncedCount = 0;
|
||||
$totalDeletedCount = 0;
|
||||
$hasSyncedFiles = false;
|
||||
$hasFailures = false;
|
||||
$travelSynced = false;
|
||||
$travelRemoteTimestamp = null;
|
||||
$snapshotResult = null;
|
||||
$orphanedDeleted = null;
|
||||
|
||||
if (null === $remoteInfo) {
|
||||
$io->warning('Remote info unavailable - will retry on next scheduled run');
|
||||
foreach ($this->syncManager->getSyncTargets() as $target) {
|
||||
$targetResult = $this->syncTarget($target, $force, $dryRun, $io, $output);
|
||||
|
||||
return Command::SUCCESS;
|
||||
if ($targetResult['failed']) {
|
||||
$hasFailures = true;
|
||||
}
|
||||
|
||||
if (false === $targetResult['synced']) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$hasSyncedFiles = true;
|
||||
$totalSyncedCount += $targetResult['updated'];
|
||||
$totalDeletedCount += $targetResult['deleted'];
|
||||
|
||||
if ($targetResult['travelSynced']) {
|
||||
$travelSynced = true;
|
||||
$travelRemoteTimestamp = $targetResult['remoteTimestamp'];
|
||||
}
|
||||
}
|
||||
|
||||
$io->text($this->formatInfoLine('Remote', $remoteInfo, $output));
|
||||
|
||||
$localInfo = $this->readLocalInfo();
|
||||
if (null !== $localInfo) {
|
||||
$io->text($this->formatInfoLine('Local', $localInfo, $output));
|
||||
} else {
|
||||
$io->text('Local: no data');
|
||||
if (false === $dryRun && $hasSyncedFiles) {
|
||||
$invalidatedTags = $this->syncManager->invalidateCaches();
|
||||
$io->text(sprintf(
|
||||
'Invalidated %d cache tags',
|
||||
$invalidatedTags,
|
||||
));
|
||||
}
|
||||
|
||||
$needsSync = $force || null === $localInfo || $remoteInfo->isNewerThan($localInfo);
|
||||
|
||||
if (false === $needsSync) {
|
||||
$io->success('Local data is up to date, no sync required');
|
||||
$this->logger->info('XML sync skipped: local data is up to date');
|
||||
|
||||
return Command::SUCCESS;
|
||||
if (false === $dryRun && $travelSynced) {
|
||||
$io->section('Refreshing snapshots');
|
||||
$snapshotResult = $this->refreshSnapshots($io);
|
||||
if (null !== $snapshotResult) {
|
||||
$orphanedDeleted = $snapshotResult->orphanedDeleted;
|
||||
}
|
||||
}
|
||||
|
||||
if ($dryRun) {
|
||||
$io->note('Sync required (dry-run mode, no files downloaded)');
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
$io->section('Syncing files');
|
||||
|
||||
try {
|
||||
$syncResult = $this->syncFiles($io);
|
||||
} catch (FilesystemException $e) {
|
||||
$io->error(sprintf('Sync failed: %s', $e->getMessage()));
|
||||
$this->logger->error('XML sync failed during file transfer', ['exception' => $e]);
|
||||
|
||||
if ($hasFailures) {
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$this->invalidateCaches();
|
||||
$io->info(sprintf('Invalidated %d cache keys', count(self::CACHE_KEYS_TO_INVALIDATE)));
|
||||
|
||||
$io->section('Refreshing snapshots');
|
||||
|
||||
$xmlFileMap = null;
|
||||
$snapshotResult = null;
|
||||
$orphanedDeleted = null;
|
||||
try {
|
||||
$xmlFileMap = $this->travelLoader->generateFilesMap();
|
||||
} catch (\Throwable $e) {
|
||||
$this->logger->warning('Failed to generate XML file map after sync; snapshot refresh skipped', [
|
||||
'error' => $e->getMessage(),
|
||||
]);
|
||||
if ($dryRun) {
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
if (null !== $xmlFileMap) {
|
||||
$total = array_sum(array_map(fn ($entry) => count($entry['hotels']), $xmlFileMap));
|
||||
$io->progressStart($total);
|
||||
|
||||
$snapshotResult = $this->travelDataService->syncSnapshotsFromXml(
|
||||
$xmlFileMap,
|
||||
function () use ($io): void { $io->progressAdvance(); },
|
||||
if ($hasSyncedFiles) {
|
||||
$successMessage = sprintf(
|
||||
'Sync complete: %d files updated, %d files deleted',
|
||||
$totalSyncedCount,
|
||||
$totalDeletedCount
|
||||
);
|
||||
|
||||
$io->progressFinish();
|
||||
if (null !== $snapshotResult) {
|
||||
$successMessage .= sprintf(
|
||||
', snapshots: %d processed, %d failed',
|
||||
$snapshotResult->processed,
|
||||
$snapshotResult->failed
|
||||
);
|
||||
}
|
||||
|
||||
$orphanedDeleted = $this->snapshotService->purgeOrphanedFutureSnapshots(array_keys($xmlFileMap));
|
||||
if (null !== $orphanedDeleted) {
|
||||
$successMessage .= sprintf(', orphaned deleted: %d', $orphanedDeleted);
|
||||
}
|
||||
|
||||
$io->success($successMessage);
|
||||
$this->logger->info('XML sync completed', [
|
||||
'files_updated' => $totalSyncedCount,
|
||||
'files_deleted' => $totalDeletedCount,
|
||||
'snapshots_processed' => $snapshotResult?->processed,
|
||||
'snapshots_failed' => $snapshotResult?->failed,
|
||||
'orphaned_deleted' => $orphanedDeleted,
|
||||
'remote_timestamp' => $travelRemoteTimestamp,
|
||||
]);
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
$successMessage = sprintf('Sync complete: %d files updated, %d files deleted', $syncResult['updated'], $syncResult['deleted']);
|
||||
if (null !== $snapshotResult) {
|
||||
$successMessage .= sprintf(', snapshots: %d processed, %d failed', $snapshotResult['processed'], $snapshotResult['failed']);
|
||||
}
|
||||
if (null !== $orphanedDeleted) {
|
||||
$successMessage .= sprintf(', orphaned deleted: %d', $orphanedDeleted);
|
||||
}
|
||||
$io->success($successMessage);
|
||||
$this->logger->info('XML sync completed', [
|
||||
'files_updated' => $syncResult['updated'],
|
||||
'files_deleted' => $syncResult['deleted'],
|
||||
'snapshots_processed' => $snapshotResult['processed'] ?? null,
|
||||
'snapshots_failed' => $snapshotResult['failed'] ?? null,
|
||||
'orphaned_deleted' => $orphanedDeleted,
|
||||
'remote_timestamp' => $remoteInfo->lastTransfer->format('c'),
|
||||
]);
|
||||
$io->success('No sync required');
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
private function readRemoteInfo(): ?XmlExportInfo
|
||||
/**
|
||||
* @return array{synced: bool, failed: bool, updated: int, deleted: int, travelSynced: bool, remoteTimestamp: ?string}
|
||||
*/
|
||||
private function syncTarget(
|
||||
BpnXmlSyncTarget $target,
|
||||
bool $force,
|
||||
bool $dryRun,
|
||||
SymfonyStyle $io,
|
||||
OutputInterface $output,
|
||||
): array
|
||||
{
|
||||
try {
|
||||
$content = $this->xmlSource->read(XmlExportInfo::getFilename());
|
||||
$datasetName = $target->name;
|
||||
|
||||
return XmlExportInfo::fromString($content);
|
||||
} catch (FilesystemException|\InvalidArgumentException $e) {
|
||||
$this->logger->warning('Could not read remote info file, likely being updated', [
|
||||
'exception' => $e->getMessage(),
|
||||
$remoteInfo = $this->syncManager->readRemoteInfo($target->source, $datasetName);
|
||||
if (null === $remoteInfo) {
|
||||
$io->warning(sprintf(
|
||||
'[%s] Remote info unavailable - will retry on next scheduled run',
|
||||
$datasetName
|
||||
));
|
||||
|
||||
return [
|
||||
'synced' => false,
|
||||
'failed' => false,
|
||||
'updated' => 0,
|
||||
'deleted' => 0,
|
||||
'travelSynced' => false,
|
||||
'remoteTimestamp' => null,
|
||||
];
|
||||
}
|
||||
|
||||
$io->text($this->formatInfoLine(sprintf('[%s] Remote', $datasetName), $remoteInfo, $output));
|
||||
|
||||
$localInfo = $this->syncManager->readLocalInfo($target->destination);
|
||||
if (null !== $localInfo) {
|
||||
$io->text($this->formatInfoLine(sprintf('[%s] Local', $datasetName), $localInfo, $output));
|
||||
} else {
|
||||
$io->text(sprintf('[%s] Local: no data', $datasetName));
|
||||
}
|
||||
|
||||
$needsSync = $force || null === $localInfo || $remoteInfo->isNewerThan($localInfo);
|
||||
if (false === $needsSync) {
|
||||
$io->text(sprintf('[%s] Local data is up to date, no sync required', $datasetName));
|
||||
$this->logger->info('XML sync skipped: local data is up to date', [
|
||||
'dataset' => $datasetName,
|
||||
]);
|
||||
|
||||
return null;
|
||||
return [
|
||||
'synced' => false,
|
||||
'failed' => false,
|
||||
'updated' => 0,
|
||||
'deleted' => 0,
|
||||
'travelSynced' => false,
|
||||
'remoteTimestamp' => null,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
private function readLocalInfo(): ?XmlExportInfo
|
||||
{
|
||||
if ($dryRun) {
|
||||
$io->note(sprintf('[%s] Sync required (dry-run mode, no files downloaded)', $datasetName));
|
||||
|
||||
return [
|
||||
'synced' => false,
|
||||
'failed' => false,
|
||||
'updated' => 0,
|
||||
'deleted' => 0,
|
||||
'travelSynced' => false,
|
||||
'remoteTimestamp' => null,
|
||||
];
|
||||
}
|
||||
|
||||
$io->section(sprintf('Syncing %s files', $datasetName));
|
||||
|
||||
$progressStarted = false;
|
||||
try {
|
||||
if (false === $this->xmlExport->fileExists(XmlExportInfo::getFilename())) {
|
||||
return null;
|
||||
$syncResult = $this->syncManager->syncFiles(
|
||||
$target->source,
|
||||
$target->destination,
|
||||
function (int $total) use ($io, &$progressStarted): void {
|
||||
$progressStarted = true;
|
||||
$io->progressStart($total);
|
||||
},
|
||||
fn () => $io->progressAdvance(),
|
||||
);
|
||||
} catch (FilesystemException $e) {
|
||||
$io->error(sprintf('[%s] Sync failed: %s', $datasetName, $e->getMessage()));
|
||||
$this->logger->error('XML sync failed during file transfer', [
|
||||
'dataset' => $datasetName,
|
||||
'exception' => $e,
|
||||
]);
|
||||
|
||||
return [
|
||||
'synced' => false,
|
||||
'failed' => true,
|
||||
'updated' => 0,
|
||||
'deleted' => 0,
|
||||
'travelSynced' => false,
|
||||
'remoteTimestamp' => null,
|
||||
];
|
||||
} finally {
|
||||
if ($progressStarted) {
|
||||
$io->progressFinish();
|
||||
}
|
||||
|
||||
$content = $this->xmlExport->read(XmlExportInfo::getFilename());
|
||||
|
||||
return XmlExportInfo::fromString($content);
|
||||
} catch (FilesystemException|\InvalidArgumentException) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, int>
|
||||
*
|
||||
* @throws FilesystemException
|
||||
*/
|
||||
private function syncFiles(SymfonyStyle $io): array
|
||||
{
|
||||
$remoteFiles = $this->xmlSource
|
||||
->listContents('.')
|
||||
->filter(fn (StorageAttributes $attributes) => $attributes->isFile())
|
||||
->map(fn (StorageAttributes $attributes) => $attributes->path())
|
||||
->toArray();
|
||||
|
||||
$localFiles = $this->xmlExport
|
||||
->listContents('.')
|
||||
->filter(fn (StorageAttributes $attributes) => $attributes->isFile())
|
||||
->map(fn (StorageAttributes $attributes) => $attributes->path())
|
||||
->toArray();
|
||||
|
||||
$filesToDelete = array_diff($localFiles, $remoteFiles);
|
||||
|
||||
$io->progressStart(count($remoteFiles));
|
||||
|
||||
foreach ($remoteFiles as $path) {
|
||||
$content = $this->xmlSource->read($path);
|
||||
$this->xmlExport->write($path, $content);
|
||||
$io->progressAdvance();
|
||||
}
|
||||
|
||||
$io->progressFinish();
|
||||
|
||||
foreach ($filesToDelete as $path) {
|
||||
$this->xmlExport->delete($path);
|
||||
if ('travel' === $datasetName) {
|
||||
$travelSynced = true;
|
||||
$remoteTimestamp = $remoteInfo->lastTransfer->format('c');
|
||||
} else {
|
||||
$travelSynced = false;
|
||||
$remoteTimestamp = null;
|
||||
}
|
||||
|
||||
$io->text(sprintf(
|
||||
'[%s] Synced %d files, deleted %d',
|
||||
$datasetName,
|
||||
$syncResult['updated'],
|
||||
$syncResult['deleted']
|
||||
));
|
||||
$this->logger->info('XML sync completed', [
|
||||
'dataset' => $datasetName,
|
||||
'files_updated' => $syncResult['updated'],
|
||||
'files_deleted' => $syncResult['deleted'],
|
||||
'remote_timestamp' => $remoteInfo->lastTransfer->format('c'),
|
||||
]);
|
||||
|
||||
return [
|
||||
'updated' => count($remoteFiles),
|
||||
'deleted' => count($filesToDelete),
|
||||
'synced' => true,
|
||||
'failed' => false,
|
||||
'updated' => $syncResult['updated'],
|
||||
'deleted' => $syncResult['deleted'],
|
||||
'travelSynced' => $travelSynced,
|
||||
'remoteTimestamp' => $remoteTimestamp,
|
||||
];
|
||||
}
|
||||
|
||||
private function invalidateCaches(): void
|
||||
private function refreshSnapshots(SymfonyStyle $io): ?BpnXmlSnapshotRefreshResult
|
||||
{
|
||||
foreach (self::CACHE_KEYS_TO_INVALIDATE as $key) {
|
||||
$this->cache->delete($key);
|
||||
$plan = $this->snapshotRefreshManager->buildPlan();
|
||||
if (null === $plan) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$progressStarted = false;
|
||||
try {
|
||||
$io->progressStart($plan->total);
|
||||
$progressStarted = true;
|
||||
|
||||
return $this->snapshotRefreshManager->refresh(
|
||||
$plan,
|
||||
fn () => $io->progressAdvance(),
|
||||
);
|
||||
} finally {
|
||||
if ($progressStarted) {
|
||||
$io->progressFinish();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user