From 07b42c9334a75ad6973c4770c611d93da20ab40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Fromme?= Date: Thu, 3 Sep 2026 10:02:36 +0200 Subject: [PATCH] feat: import buspro xml files from remote sftp source --- composer.json | 1 + composer.lock | 229 +++++++- .../Classes/Command/DateCommandController.php | 21 +- .../Classes/Service/DateImportService.php | 41 +- .../Classes/Service/LocalImportSource.php | 49 ++ .../Classes/Service/ProductImportSource.php | 30 + .../Service/ProductImportSourceFactory.php | 35 ++ .../Classes/Service/SftpImportException.php | 7 + .../Classes/Service/SftpImportSource.php | 532 ++++++++++++++++++ .../Classes/Task/ImportDatesTask.php | 14 +- .../ext/ep_products/ext_conf_template.txt | 24 + 11 files changed, 954 insertions(+), 29 deletions(-) create mode 100644 public/typo3conf/ext/ep_products/Classes/Service/LocalImportSource.php create mode 100644 public/typo3conf/ext/ep_products/Classes/Service/ProductImportSource.php create mode 100644 public/typo3conf/ext/ep_products/Classes/Service/ProductImportSourceFactory.php create mode 100644 public/typo3conf/ext/ep_products/Classes/Service/SftpImportException.php create mode 100644 public/typo3conf/ext/ep_products/Classes/Service/SftpImportSource.php diff --git a/composer.json b/composer.json index 03a73104..0b995bff 100644 --- a/composer.json +++ b/composer.json @@ -29,6 +29,7 @@ "league/oauth2-client": "^2.8", "league/period": "^4.9", "lochmueller/staticfilecache": "^12.5", + "phpseclib/phpseclib": "^3.0", "sjbr/sr-freecap": "2.6.0", "ssch/typo3-encore": "^3.0", "symfony/http-client": "^4.3", diff --git a/composer.lock b/composer.lock index bfe9f287..4d7f9f16 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f0ff2587673ce0da1d0d53cc382a0922", + "content-hash": "0842a6e9476f96ae559c7d2e900c3be2", "packages": [ { "name": "b13/container", @@ -2171,6 +2171,123 @@ }, "time": "2025-12-06T11:45:25+00:00" }, + { + "name": "paragonie/constant_time_encoding", + "version": "v2.8.2", + "source": { + "type": "git", + "url": "https://github.com/paragonie/constant_time_encoding.git", + "reference": "e30811f7bc69e4b5b6d5783e712c06c8eabf0226" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/e30811f7bc69e4b5b6d5783e712c06c8eabf0226", + "reference": "e30811f7bc69e4b5b6d5783e712c06c8eabf0226", + "shasum": "" + }, + "require": { + "php": "^7|^8" + }, + "require-dev": { + "phpunit/phpunit": "^6|^7|^8|^9", + "vimeo/psalm": "^1|^2|^3|^4" + }, + "type": "library", + "autoload": { + "psr-4": { + "ParagonIE\\ConstantTime\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Paragon Initiative Enterprises", + "email": "security@paragonie.com", + "homepage": "https://paragonie.com", + "role": "Maintainer" + }, + { + "name": "Steve 'Sc00bz' Thomas", + "email": "steve@tobtu.com", + "homepage": "https://www.tobtu.com", + "role": "Original Developer" + } + ], + "description": "Constant-time Implementations of RFC 4648 Encoding (Base-64, Base-32, Base-16)", + "keywords": [ + "base16", + "base32", + "base32_decode", + "base32_encode", + "base64", + "base64_decode", + "base64_encode", + "bin2hex", + "encoding", + "hex", + "hex2bin", + "rfc4648" + ], + "support": { + "email": "info@paragonie.com", + "issues": "https://github.com/paragonie/constant_time_encoding/issues", + "source": "https://github.com/paragonie/constant_time_encoding" + }, + "time": "2025-09-24T15:12:37+00:00" + }, + { + "name": "paragonie/random_compat", + "version": "v9.99.100", + "source": { + "type": "git", + "url": "https://github.com/paragonie/random_compat.git", + "reference": "996434e5492cb4c3edcb9168db6fbb1359ef965a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/paragonie/random_compat/zipball/996434e5492cb4c3edcb9168db6fbb1359ef965a", + "reference": "996434e5492cb4c3edcb9168db6fbb1359ef965a", + "shasum": "" + }, + "require": { + "php": ">= 7" + }, + "require-dev": { + "phpunit/phpunit": "4.*|5.*", + "vimeo/psalm": "^1" + }, + "suggest": { + "ext-libsodium": "Provides a modern crypto API that can be used to generate random bytes." + }, + "type": "library", + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Paragon Initiative Enterprises", + "email": "security@paragonie.com", + "homepage": "https://paragonie.com" + } + ], + "description": "PHP 5.x polyfill for random_bytes() and random_int() from PHP 7", + "keywords": [ + "csprng", + "polyfill", + "pseudorandom", + "random" + ], + "support": { + "email": "info@paragonie.com", + "issues": "https://github.com/paragonie/random_compat/issues", + "source": "https://github.com/paragonie/random_compat" + }, + "time": "2020-10-15T08:29:30+00:00" + }, { "name": "phpdocumentor/reflection-common", "version": "2.2.0", @@ -2346,6 +2463,116 @@ }, "time": "2025-11-21T15:09:14+00:00" }, + { + "name": "phpseclib/phpseclib", + "version": "3.0.57", + "source": { + "type": "git", + "url": "https://github.com/phpseclib/phpseclib.git", + "reference": "d17e0ddaeaf6f22f7e007cbb437d78792fe2a0e4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/d17e0ddaeaf6f22f7e007cbb437d78792fe2a0e4", + "reference": "d17e0ddaeaf6f22f7e007cbb437d78792fe2a0e4", + "shasum": "" + }, + "require": { + "paragonie/constant_time_encoding": "^1|^2|^3", + "paragonie/random_compat": "^1.4|^2.0|^9.99.99", + "php": ">=5.6.1" + }, + "require-dev": { + "phpunit/phpunit": "*" + }, + "suggest": { + "ext-dom": "Install the DOM extension to load XML formatted public keys.", + "ext-gmp": "Install the GMP (GNU Multiple Precision) extension in order to speed up arbitrary precision integer arithmetic operations.", + "ext-libsodium": "SSH2/SFTP can make use of some algorithms provided by the libsodium-php extension.", + "ext-mcrypt": "Install the Mcrypt extension in order to speed up a few other cryptographic operations.", + "ext-openssl": "Install the OpenSSL extension in order to speed up a wide variety of cryptographic operations." + }, + "type": "library", + "autoload": { + "files": [ + "phpseclib/bootstrap.php" + ], + "psr-4": { + "phpseclib3\\": "phpseclib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jim Wigginton", + "email": "terrafrost@php.net", + "role": "Lead Developer" + }, + { + "name": "Patrick Monnerat", + "email": "pm@datasphere.ch", + "role": "Developer" + }, + { + "name": "Andreas Fischer", + "email": "bantu@phpbb.com", + "role": "Developer" + }, + { + "name": "Hans-Jürgen Petrich", + "email": "petrich@tronic-media.com", + "role": "Developer" + }, + { + "name": "Graham Campbell", + "email": "graham@alt-three.com", + "role": "Developer" + } + ], + "description": "PHP Secure Communications Library - Pure-PHP implementations of RSA, AES, SSH2, SFTP, X.509 etc.", + "homepage": "http://phpseclib.sourceforge.net", + "keywords": [ + "BigInteger", + "aes", + "asn.1", + "asn1", + "blowfish", + "crypto", + "cryptography", + "encryption", + "rsa", + "security", + "sftp", + "signature", + "signing", + "ssh", + "twofish", + "x.509", + "x509" + ], + "support": { + "issues": "https://github.com/phpseclib/phpseclib/issues", + "source": "https://github.com/phpseclib/phpseclib/tree/3.0.57" + }, + "funding": [ + { + "url": "https://github.com/terrafrost", + "type": "github" + }, + { + "url": "https://www.patreon.com/phpseclib", + "type": "patreon" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpseclib/phpseclib", + "type": "tidelift" + } + ], + "time": "2026-08-26T12:13:21+00:00" + }, { "name": "phpstan/phpdoc-parser", "version": "2.3.3", diff --git a/public/typo3conf/ext/ep_products/Classes/Command/DateCommandController.php b/public/typo3conf/ext/ep_products/Classes/Command/DateCommandController.php index fafe871b..450c58a7 100644 --- a/public/typo3conf/ext/ep_products/Classes/Command/DateCommandController.php +++ b/public/typo3conf/ext/ep_products/Classes/Command/DateCommandController.php @@ -28,10 +28,10 @@ namespace EP\EpProducts\Command; ***************************************************************/ use EP\EpProducts\Service\DateImportService; +use EP\EpProducts\Service\ProductImportSourceFactory; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; -use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; use TYPO3\CMS\Extbase\Service\CacheService; @@ -53,21 +53,29 @@ class DateCommandController extends Command */ protected $configurationManager; + /** + * @var ProductImportSourceFactory + */ + protected $importSourceFactory; + /** * @param DateImportService $importService * @param CacheService $cacheService * @param ConfigurationManagerInterface $configurationManager + * @param ProductImportSourceFactory $importSourceFactory */ public function __construct ( DateImportService $importService, CacheService $cacheService, - ConfigurationManagerInterface $configurationManager + ConfigurationManagerInterface $configurationManager, + ProductImportSourceFactory $importSourceFactory ) { $this->dateImportService = $importService; $this->cacheService = $cacheService; $this->configurationManager = $configurationManager; + $this->importSourceFactory = $importSourceFactory; parent::__construct(); } @@ -79,8 +87,15 @@ class DateCommandController extends Command */ public function execute(InputInterface $input, OutputInterface $output): int { - $path = GeneralUtility::getFileAbsFileName('fileadmin/xmlexport'); + $importSource = $this->importSourceFactory->get(); + $path = $importSource->acquire(); + if ($path === null) { + $output->writeln('No new import available.'); + return 0; + } + $count = $this->dateImportService->import($path); + $importSource->markImported($path); $output->writeln(sprintf('%d rows imported.', $count )); $settings = $this->getSettings(); $resellerPageUid = $settings['resellerExportPageUid']; diff --git a/public/typo3conf/ext/ep_products/Classes/Service/DateImportService.php b/public/typo3conf/ext/ep_products/Classes/Service/DateImportService.php index 91a7b271..da6c12de 100644 --- a/public/typo3conf/ext/ep_products/Classes/Service/DateImportService.php +++ b/public/typo3conf/ext/ep_products/Classes/Service/DateImportService.php @@ -85,6 +85,11 @@ class DateImportService implements SingletonInterface, LoggerAwareInterface */ protected $cacheService; + /** + * @var ProductImportSourceFactory + */ + protected $importSourceFactory; + /** * @var array */ @@ -95,10 +100,16 @@ class DateImportService implements SingletonInterface, LoggerAwareInterface /** * @param ConfigurationManagerInterface $configurationManager * @param CacheService $cacheService + * @param ProductImportSourceFactory $importSourceFactory */ - public function __construct(ConfigurationManagerInterface $configurationManager, CacheService $cacheService) { + public function __construct( + ConfigurationManagerInterface $configurationManager, + CacheService $cacheService, + ProductImportSourceFactory $importSourceFactory + ) { $this->configurationManager = $configurationManager; $this->cacheService = $cacheService; + $this->importSourceFactory = $importSourceFactory; $settings = GeneralUtility::removeDotsFromTS( $this->configurationManager->getConfiguration(ConfigurationManagerInterface::CONFIGURATION_TYPE_FULL_TYPOSCRIPT) @@ -121,7 +132,7 @@ class DateImportService implements SingletonInterface, LoggerAwareInterface $this->db = $this->getDbConnection(); $this->getRoomMappings(); $this->getHotelMappings(); - $this->importPickups(); + $this->importPickups($path); $this->createTempTables(); $this->logger->info('starting product import'); $dateCount = $this->parseXmlFilesForProducts($path); @@ -135,24 +146,7 @@ class DateImportService implements SingletonInterface, LoggerAwareInterface public function isImportRequired(): bool { - $timestampService = new ImportTimestampService(); - - return $timestampService->isImportRequired( - 'fileadmin/xmlexport/uebertragung.info', - 'fileadmin/products_import.info' - ); - } - - /** - * @param $path - * - * @return bool - */ - public function checkActiveUpload($path): bool - { - $tempFile = $path . '/.pureftpd-upload.*'; - - return count(glob($tempFile)) > 0; + return $this->importSourceFactory->get()->isNewImportAvailable(); } public function parseXmlFilesForProducts($path): int @@ -877,12 +871,13 @@ class DateImportService implements SingletonInterface, LoggerAwareInterface } } - protected function importPickups(): void + protected function importPickups(string $path): void { libxml_use_internal_errors (true); - $path = GeneralUtility::getFileAbsFileName('fileadmin/xmlexport/zustiege.xml'); - if (!$xmlData = simplexml_load_string(file_get_contents($path))) { + $file = $path . '/zustiege.xml'; + if (!is_file($file) || !$xmlData = simplexml_load_string((string) file_get_contents($file))) { libxml_clear_errors(); + $this->logger->warning('pickup xml missing or invalid', ['file' => $file]); return; } diff --git a/public/typo3conf/ext/ep_products/Classes/Service/LocalImportSource.php b/public/typo3conf/ext/ep_products/Classes/Service/LocalImportSource.php new file mode 100644 index 00000000..8c71adfc --- /dev/null +++ b/public/typo3conf/ext/ep_products/Classes/Service/LocalImportSource.php @@ -0,0 +1,49 @@ +isNewImportAvailable()) { + return null; + } + + return GeneralUtility::getFileAbsFileName(self::XML_DIR); + } + + public function markImported(string $path): void + { + (new ImportTimestampService())->writeImportTimestamp(self::IMPORT_MARKER); + } + + public function isNewImportAvailable(): bool + { + $timestampService = new ImportTimestampService(); + + $uploadMarkerPath = GeneralUtility::getFileAbsFileName(self::UPLOAD_MARKER); + if ($uploadMarkerPath === '' || !is_file($uploadMarkerPath)) { + // No supplier marker at all: import whatever is in the folder (legacy behaviour). + return true; + } + + $uploadedAt = $timestampService->readImportTimestamp(self::UPLOAD_MARKER); + $lastImportedAt = $timestampService->readImportTimestamp(self::IMPORT_MARKER); + + return $uploadedAt === null || $lastImportedAt === null || $uploadedAt > $lastImportedAt; + } +} diff --git a/public/typo3conf/ext/ep_products/Classes/Service/ProductImportSource.php b/public/typo3conf/ext/ep_products/Classes/Service/ProductImportSource.php new file mode 100644 index 00000000..d453f377 --- /dev/null +++ b/public/typo3conf/ext/ep_products/Classes/Service/ProductImportSource.php @@ -0,0 +1,30 @@ +getMode() === self::MODE_LOCAL) { + return GeneralUtility::makeInstance(LocalImportSource::class); + } + + return GeneralUtility::makeInstance(SftpImportSource::class); + } + + public function getMode(): string + { + $config = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('ep_products'); + $mode = strtolower(trim((string)(is_array($config) ? ($config['productImportSource'] ?? '') : ''))); + + return $mode === self::MODE_LOCAL ? self::MODE_LOCAL : self::MODE_SFTP; + } +} diff --git a/public/typo3conf/ext/ep_products/Classes/Service/SftpImportException.php b/public/typo3conf/ext/ep_products/Classes/Service/SftpImportException.php new file mode 100644 index 00000000..3b6c51cd --- /dev/null +++ b/public/typo3conf/ext/ep_products/Classes/Service/SftpImportException.php @@ -0,0 +1,7 @@ +fetchIfNewer(); + } + + /** + * {@see ProductImportSource::markImported()} — publish the download as the new + * change-detection baseline (only reached after a successful import), then + * refresh the display-only marker for the BE module and system-info toolbar. + */ + public function markImported(string $path): void + { + $this->commit($path); + (new ImportTimestampService())->writeImportTimestamp(self::IMPORT_MARKER); + } + + /** + * {@see ProductImportSource::isNewImportAvailable()} — alias for {@see isRemoteNewer()}. + */ + public function isNewImportAvailable(): bool + { + return $this->isRemoteNewer(); + } + + /** + * Timestamp of the export currently offered on the remote server, or null when + * the marker is absent/unreadable/unparseable (supplier upload not finished). + */ + public function getRemoteUploadTimestamp(): ?\DateTime + { + $config = $this->buildConfig(); + $sftp = $this->connect($config); + try { + [, $markerBytes] = $this->readRemoteMarker($sftp, $config['remotePath']); + } finally { + $sftp->disconnect(); + } + + return $markerBytes !== null ? $this->parseMarkerTimestamp($markerBytes) : null; + } + + /** + * Timestamp of the export we last successfully imported (staged marker copy). + */ + public function getStagedImportedTimestamp(): ?\DateTime + { + $marker = $this->stagingBasePath() . '/' . self::PUBLISHED_DIR . '/' . self::MARKER_FILE; + if (!is_file($marker)) { + return null; + } + + return $this->parseMarkerTimestamp((string)file_get_contents($marker)); + } + + /** + * True when the remote export is newer than the last imported one (or nothing + * has been imported yet). False when there is nothing to do. + */ + public function isRemoteNewer(): bool + { + $remoteTs = $this->getRemoteUploadTimestamp(); + if ($remoteTs === null) { + return false; + } + + $stagedTs = $this->getStagedImportedTimestamp(); + + return $stagedTs === null || $remoteTs > $stagedTs; + } + + /** + * Freshness check + (only when newer) full download into a fresh temp dir. + * Returns the temp-dir path — NOT yet published. The caller runs + * DateImportService::import() against it and calls {@see commit()} on success. + * Returns null when the remote export is not newer / not ready. + * + * @throws SftpImportException on connect/auth/host-key/partial-transfer failure + */ + public function fetchIfNewer(): ?string + { + $this->cleanupStale(); + $config = $this->buildConfig(); + $sftp = $this->connect($config); + + try { + [$remoteDir, $markerBytes] = $this->readRemoteMarker($sftp, $config['remotePath']); + $remoteTs = $markerBytes !== null ? $this->parseMarkerTimestamp($markerBytes) : null; + + if ($remoteTs === null) { + $this->logWarning('remote upload marker missing or unparseable; skipping product import', [ + 'host' => $config['host'], + 'remotePath' => $config['remotePath'], + ]); + + return null; + } + + $stagedTs = $this->getStagedImportedTimestamp(); + if ($stagedTs !== null && $remoteTs <= $stagedTs) { + return null; + } + + return $this->downloadInto($sftp, $remoteDir, $markerBytes); + } finally { + $sftp->disconnect(); + } + } + + /** + * Unconditional download into a fresh temp dir. Returns the temp-dir path. + * + * @throws SftpImportException + */ + public function fetch(): string + { + $this->cleanupStale(); + $config = $this->buildConfig(); + $sftp = $this->connect($config); + + try { + [$remoteDir, $markerBytes] = $this->readRemoteMarker($sftp, $config['remotePath']); + if ($markerBytes === null) { + throw new SftpImportException( + 'Remote export has no readable ' . self::MARKER_FILE . ' under "' . $config['remotePath'] . '"' + ); + } + + return $this->downloadInto($sftp, $remoteDir, $markerBytes); + } finally { + $sftp->disconnect(); + } + } + + /** + * Publish a temp dir produced by {@see fetchIfNewer()} / {@see fetch()} to + * .../current, making its marker the new change-detection baseline. Call only + * after import() succeeded. + * + * @throws SftpImportException + */ + public function commit(string $stagedPath): string + { + $base = $this->stagingBasePath(); + $stagedPath = rtrim($stagedPath, '/'); + + if (strpos($stagedPath . '/', $base . '/') !== 0 || !is_dir($stagedPath)) { + throw new SftpImportException('Refusing to publish unknown staging directory: ' . $stagedPath); + } + + $published = $base . '/' . self::PUBLISHED_DIR; + + if (is_dir($published)) { + $retired = $published . '.old-' . time() . '-' . bin2hex(random_bytes(3)); + if (!@rename($published, $retired)) { + throw new SftpImportException('Cannot move previous import aside: ' . $published); + } + } + + if (!@rename($stagedPath, $published)) { + throw new SftpImportException('Cannot publish staging directory to ' . $published); + } + + foreach ((array)glob($base . '/' . self::PUBLISHED_DIR . '.old-*', GLOB_ONLYDIR) as $old) { + GeneralUtility::rmdir($old, true); + } + + return $published; + } + + /** + * @param array{host:string,port:int,user:string,password:string,remotePath:string,timeout:int,hostKey:string} $config + */ + private function connect(array $config): SFTP + { + try { + $sftp = new SFTP($config['host'], $config['port'], $config['timeout']); + } catch (\Throwable $e) { + throw new SftpImportException( + sprintf('Cannot connect to SFTP host %s:%d: %s', $config['host'], $config['port'], $e->getMessage()), + 0, + $e + ); + } + + if ($config['hostKey'] !== '') { + $this->verifyHostKey($sftp, $config['host'], $config['hostKey']); + } + + try { + $authenticated = $sftp->login($config['user'], $config['password']); + } catch (\Throwable $e) { + throw new SftpImportException( + sprintf('Cannot connect to SFTP host %s:%d: %s', $config['host'], $config['port'], $e->getMessage()), + 0, + $e + ); + } + + if ($authenticated !== true) { + throw new SftpImportException(sprintf('SFTP authentication failed for user "%s"', $config['user'])); + } + + return $sftp; + } + + private function verifyHostKey(SFTP $sftp, string $host, string $expected): void + { + try { + $actual = $sftp->getServerPublicHostKey(); + } catch (\Throwable $e) { + throw new SftpImportException('Cannot read SSH host key for ' . $host . ': ' . $e->getMessage(), 0, $e); + } + + if ($actual === false) { + throw new SftpImportException('Cannot read SSH host key for ' . $host); + } + + $expected = trim($expected); + $actual = trim($actual); + + if (hash_equals($actual, $expected)) { + return; + } + + $parts = explode(' ', $actual); + $rawKey = isset($parts[1]) ? base64_decode($parts[1], true) : false; + if ($rawKey !== false) { + $fingerprint = 'SHA256:' . rtrim(base64_encode(hash('sha256', $rawKey, true)), '='); + if (hash_equals($fingerprint, $expected)) { + return; + } + } + + throw new SftpImportException('SSH host key mismatch for ' . $host . ' — refusing to connect'); + } + + /** + * Reads the upload marker, probing each acceptable form of the configured + * remote directory. Some SFTP servers (e.g. Hetzner Storage Box) only accept + * paths relative to the login directory and silently fail on a leading slash. + * + * @return array{0:string,1:?string} the directory form that worked, and the marker bytes (null if none did) + */ + private function readRemoteMarker(SFTP $sftp, string $configuredPath): array + { + $candidates = $this->remoteDirCandidates($configuredPath); + + foreach ($candidates as $dir) { + try { + $contents = $sftp->get($this->joinRemote($dir, self::MARKER_FILE)); + } catch (\Throwable $e) { + throw new SftpImportException( + 'Failed to read remote ' . self::MARKER_FILE . ': ' . $e->getMessage(), + 0, + $e + ); + } + + if ($contents !== false) { + return [$dir, (string)$contents]; + } + } + + return [$candidates[0], null]; + } + + /** + * @return string[] non-empty, unique + */ + private function remoteDirCandidates(string $configuredPath): array + { + $configured = trim($configuredPath); + $relative = ltrim($configured, '/'); + + $candidates = []; + if ($configured !== '' && $configured !== '/') { + $candidates[] = $configured; + } + if ($relative !== '' && $relative !== $configured) { + $candidates[] = $relative; + } + $candidates[] = '.'; + + return array_values(array_unique($candidates)); + } + + /** + * Download Ziel_*.xml + zustiege.xml + uebertragung.info into a fresh temp dir, + * verifying each file's size against the remote listing. The marker is written + * last, from the bytes already fetched for the freshness check. + * + * @throws SftpImportException + */ + private function downloadInto(SFTP $sftp, string $remotePath, string $markerBytes): string + { + $list = $sftp->rawlist($remotePath, false); + if ($list === false) { + throw new SftpImportException('Cannot list remote directory ' . $remotePath); + } + + $productFiles = []; + $pickupSize = null; + foreach ($list as $name => $attrs) { + if ($name === '.' || $name === '..' || !is_array($attrs)) { + continue; + } + if ((int)($attrs['type'] ?? 0) === self::REMOTE_TYPE_DIRECTORY) { + continue; + } + if (strpos($name, self::PRODUCT_PREFIX) === 0 && substr($name, -4) === '.xml') { + $productFiles[$name] = (int)($attrs['size'] ?? -1); + } elseif ($name === self::PICKUPS_FILE) { + $pickupSize = (int)($attrs['size'] ?? -1); + } + } + + if (count($productFiles) === 0) { + throw new SftpImportException('Remote export contains no ' . self::PRODUCT_PREFIX . '_*.xml files'); + } + + $previousCount = $this->stagedProductFileCount(); + if ($previousCount > 0 && count($productFiles) < (int)floor($previousCount * 0.5)) { + throw new SftpImportException(sprintf( + 'Remote export looks truncated (%d %s_*.xml files vs %d previously) — refusing to import', + count($productFiles), + self::PRODUCT_PREFIX, + $previousCount + )); + } + + $toDownload = $productFiles; + if ($pickupSize !== null) { + $toDownload[self::PICKUPS_FILE] = $pickupSize; + } else { + $this->logWarning('remote export has no ' . self::PICKUPS_FILE . '; pickups will be skipped'); + } + + $tmp = $this->newTempDir(); + + foreach ($toDownload as $name => $expectedSize) { + $target = $tmp . '/' . $name; + try { + $ok = $sftp->get($this->joinRemote($remotePath, $name), $target); + } catch (\Throwable $e) { + GeneralUtility::rmdir($tmp, true); + throw new SftpImportException('Download failed for ' . $name . ': ' . $e->getMessage(), 0, $e); + } + + if ($ok === false || !is_file($target)) { + GeneralUtility::rmdir($tmp, true); + throw new SftpImportException('Download failed for ' . $name); + } + + clearstatcache(true, $target); + if ($expectedSize >= 0 && filesize($target) !== $expectedSize) { + $actualSize = filesize($target); + GeneralUtility::rmdir($tmp, true); + throw new SftpImportException(sprintf( + 'Size mismatch for %s (got %d bytes, expected %d)', + $name, + $actualSize, + $expectedSize + )); + } + } + + // Write the completion marker last, mirroring the supplier's upload order. + GeneralUtility::writeFile($tmp . '/' . self::MARKER_FILE, $markerBytes, false); + + return $tmp; + } + + private function stagedProductFileCount(): int + { + $dir = $this->stagingBasePath() . '/' . self::PUBLISHED_DIR; + if (!is_dir($dir)) { + return 0; + } + + $count = 0; + foreach ((array)scandir($dir) as $name) { + if (strpos((string)$name, self::PRODUCT_PREFIX) === 0 && substr((string)$name, -4) === '.xml') { + $count++; + } + } + + return $count; + } + + private function parseMarkerTimestamp(string $contents): ?\DateTime + { + $line = trim((string)strtok($contents, "\r\n")); + if ($line === '') { + return null; + } + + $date = \DateTime::createFromFormat(self::MARKER_TS_FORMAT, $line); + + return $date instanceof \DateTime ? $date : null; + } + + private function logWarning(string $message, array $context = []): void + { + if ($this->logger !== null) { + $this->logger->warning($message, $context); + } + } + + private function stagingBasePath(): string + { + return Environment::getVarPath() . '/transient/' . self::STAGING_SUBDIR; + } + + private function newTempDir(): string + { + $dir = $this->stagingBasePath() . '/.tmp-' . getmypid() . '-' . bin2hex(random_bytes(4)); + GeneralUtility::mkdir_deep($dir); + if (!is_dir($dir) || !is_writable($dir)) { + throw new SftpImportException('Cannot create staging directory: ' . $dir); + } + + return $dir; + } + + private function cleanupStale(): void + { + $base = $this->stagingBasePath(); + if (!is_dir($base)) { + return; + } + + $stale = array_merge( + (array)glob($base . '/.tmp-*', GLOB_ONLYDIR), + (array)glob($base . '/' . self::PUBLISHED_DIR . '.old-*', GLOB_ONLYDIR) + ); + foreach ($stale as $dir) { + GeneralUtility::rmdir($dir, true); + } + } + + private function joinRemote(string $base, string $name): string + { + return rtrim($base, '/') . '/' . ltrim($name, '/'); + } + + /** + * @return array{host:string,port:int,user:string,password:string,remotePath:string,timeout:int,hostKey:string} + */ + private function buildConfig(): array + { + if ($this->configuration === null) { + $raw = GeneralUtility::makeInstance(ExtensionConfiguration::class)->get('ep_products'); + $raw = is_array($raw) ? $raw : []; + + $host = trim((string)($raw['productImportSftpHost'] ?? '')); + $user = trim((string)($raw['productImportSftpUser'] ?? '')); + $password = (string)($raw['productImportSftpPassword'] ?? ''); + $remotePath = trim((string)($raw['productImportSftpRemotePath'] ?? '/')); + $port = (int)($raw['productImportSftpPort'] ?? 22); + $timeout = (int)($raw['productImportSftpTimeout'] ?? 15); + $hostKey = trim((string)($raw['productImportSftpHostKey'] ?? '')); + + if ($host === '' || $user === '') { + throw new SftpImportException( + 'SFTP product import is not configured: host and username are required ' + . '(Admin Tools → Settings → Extension Configuration → ep_products).' + ); + } + if ($password === '') { + throw new SftpImportException('SFTP product import is not configured: password is required.'); + } + + $this->configuration = [ + 'host' => $host, + 'port' => $port > 0 ? $port : 22, + 'user' => $user, + 'password' => $password, + 'remotePath' => $remotePath === '' ? '/' : $remotePath, + 'timeout' => $timeout > 0 ? $timeout : 15, + 'hostKey' => $hostKey, + ]; + } + + return $this->configuration; + } +} diff --git a/public/typo3conf/ext/ep_products/Classes/Task/ImportDatesTask.php b/public/typo3conf/ext/ep_products/Classes/Task/ImportDatesTask.php index 572c25de..05adac25 100644 --- a/public/typo3conf/ext/ep_products/Classes/Task/ImportDatesTask.php +++ b/public/typo3conf/ext/ep_products/Classes/Task/ImportDatesTask.php @@ -28,6 +28,7 @@ namespace EP\EpProducts\Task; ***************************************************************/ use EP\EpProducts\Service\DateImportService; +use EP\EpProducts\Service\ProductImportSourceFactory; use TYPO3\CMS\Core\Utility\GeneralUtility; use TYPO3\CMS\Extbase\Configuration\ConfigurationManager; use TYPO3\CMS\Extbase\Configuration\ConfigurationManagerInterface; @@ -39,11 +40,20 @@ class ImportDatesTask extends AbstractTask public function execute(): bool { $importService = GeneralUtility::makeInstance(DateImportService::class); + $importSource = GeneralUtility::makeInstance(ProductImportSourceFactory::class)->get(); $cacheService = GeneralUtility::makeInstance(CacheService::class); - if ($importService->isImportRequired()) { - $path = GeneralUtility::getFileAbsFileName('fileadmin/xmlexport'); + // Acquire the export only when it is newer than the one we last imported + // (SFTP mode: download to a staging dir; local mode: fileadmin/xmlexport). + // An acquisition failure (SFTP connect/auth/transfer) is left to propagate + // so the scheduler marks the task failed. + $path = $importSource->acquire(); + + if ($path !== null) { $importService->import($path); + // Advance the "last imported" bookkeeping only after a successful + // import, so a crashed import retries on the next run. + $importSource->markImported($path); } $settings = $this->getSettings(); diff --git a/public/typo3conf/ext/ep_products/ext_conf_template.txt b/public/typo3conf/ext/ep_products/ext_conf_template.txt index 0e7d3be8..7ed08e6a 100644 --- a/public/typo3conf/ext/ep_products/ext_conf_template.txt +++ b/public/typo3conf/ext/ep_products/ext_conf_template.txt @@ -39,3 +39,27 @@ bpnConnectApiTimeout = 5 # cat=MyEpAPI; type=int; label=Bpn Connect API max total request duration in seconds (0 = unlimited) bpnConnectApiMaxDuration = 15 + +# cat=SftpImport; type=options[Remote SFTP server=sftp,Local directory (fileadmin/xmlexport)=local]; label=Product/date import source +productImportSource = sftp + +# cat=SftpImport; type=string; label=SFTP host for product/date import +productImportSftpHost = + +# cat=SftpImport; type=int+; label=SFTP port +productImportSftpPort = 22 + +# cat=SftpImport; type=string; label=SFTP username +productImportSftpUser = + +# cat=SftpImport; type=string; label=SFTP password +productImportSftpPassword = + +# cat=SftpImport; type=string; label=Remote directory holding Ziel_*.xml, zustiege.xml and uebertragung.info +productImportSftpRemotePath = / + +# cat=SftpImport; type=string; label=Expected SSH host key (optional; "ssh-ed25519 AAAA..." or "SHA256:..."; empty disables the check) +productImportSftpHostKey = + +# cat=SftpImport; type=int+; label=SFTP connection timeout in seconds +productImportSftpTimeout = 15