diff --git a/composer.json b/composer.json index 9453fd4..6e1e7a8 100644 --- a/composer.json +++ b/composer.json @@ -128,6 +128,7 @@ }, "require-dev": { "deployer/deployer": "^7.3", + "fakerphp/faker": "*", "friendsofphp/php-cs-fixer": "^3.84", "marcocesarato/php-conventional-changelog": "^1.17", "phpunit/phpunit": "^9.5", diff --git a/composer.lock b/composer.lock index 0c49372..5d58b4f 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "f0504df1803c80691eea135d9f5b7ddc", + "content-hash": "855c5c3b38925411e3e8626995e74076", "packages": [ { "name": "beberlei/doctrineextensions", @@ -11236,6 +11236,69 @@ }, "time": "2023-08-08T05:53:35+00:00" }, + { + "name": "fakerphp/faker", + "version": "v1.24.1", + "source": { + "type": "git", + "url": "https://github.com/FakerPHP/Faker.git", + "reference": "e0ee18eb1e6dc3cda3ce9fd97e5a0689a88a64b5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/FakerPHP/Faker/zipball/e0ee18eb1e6dc3cda3ce9fd97e5a0689a88a64b5", + "reference": "e0ee18eb1e6dc3cda3ce9fd97e5a0689a88a64b5", + "shasum": "" + }, + "require": { + "php": "^7.4 || ^8.0", + "psr/container": "^1.0 || ^2.0", + "symfony/deprecation-contracts": "^2.2 || ^3.0" + }, + "conflict": { + "fzaninotto/faker": "*" + }, + "require-dev": { + "bamarni/composer-bin-plugin": "^1.4.1", + "doctrine/persistence": "^1.3 || ^2.0", + "ext-intl": "*", + "phpunit/phpunit": "^9.5.26", + "symfony/phpunit-bridge": "^5.4.16" + }, + "suggest": { + "doctrine/orm": "Required to use Faker\\ORM\\Doctrine", + "ext-curl": "Required by Faker\\Provider\\Image to download images.", + "ext-dom": "Required by Faker\\Provider\\HtmlLorem for generating random HTML.", + "ext-iconv": "Required by Faker\\Provider\\ru_RU\\Text::realText() for generating real Russian text.", + "ext-mbstring": "Required for multibyte Unicode string functionality." + }, + "type": "library", + "autoload": { + "psr-4": { + "Faker\\": "src/Faker/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "François Zaninotto" + } + ], + "description": "Faker is a PHP library that generates fake data for you.", + "keywords": [ + "data", + "faker", + "fixtures" + ], + "support": { + "issues": "https://github.com/FakerPHP/Faker/issues", + "source": "https://github.com/FakerPHP/Faker/tree/v1.24.1" + }, + "time": "2024-11-21T13:46:39+00:00" + }, { "name": "fidry/cpu-core-counter", "version": "1.3.0", diff --git a/config/services.yaml b/config/services.yaml index 7122263..41e07cf 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -61,6 +61,14 @@ services: arguments: $connection: '@doctrine.dbal.default_connection' + App\Command\DbAnonymizeCommand: + arguments: + $environment: '%kernel.environment%' + + App\Service\Common\DatabaseAnonymizer: + arguments: + $connection: '@doctrine.dbal.default_connection' + App\BusProNet\ApiClient: arguments: $logger: '@monolog.logger.bpn' diff --git a/src/Command/DbAnonymizeCommand.php b/src/Command/DbAnonymizeCommand.php new file mode 100644 index 0000000..2639cf5 --- /dev/null +++ b/src/Command/DbAnonymizeCommand.php @@ -0,0 +1,89 @@ +addOption('dry-run', null, InputOption::VALUE_NONE, 'Report the affected rows without writing anything') + ->addOption('seed', null, InputOption::VALUE_REQUIRED, 'Seed for the synthetic data, so repeated runs produce identical output') + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + if ('prod' === $this->environment) { + $io->error('The database anonymizer is disabled in the prod environment.'); + + return Command::FAILURE; + } + + $dryRun = true === $input->getOption('dry-run'); + $seed = $input->getOption('seed'); + + if (false === $dryRun && true === $input->isInteractive() && false === $io->confirm('This irreversibly overwrites personal data in the current database. Continue?', false)) { + $io->warning('Aborted.'); + + return Command::SUCCESS; + } + + try { + $report = null === $seed + ? $this->databaseAnonymizer->anonymizeAll($dryRun) + : $this->databaseAnonymizer->anonymizeAll($dryRun, (int) $seed); + } catch (\Throwable $exception) { + $this->logger->error('Database anonymization failed', [ + 'environment' => $this->environment, + 'error' => $exception->getMessage(), + ]); + + $io->error($exception->getMessage()); + + return Command::FAILURE; + } + + $rows = []; + foreach ($report as $table => $count) { + $rows[] = [$table, $count]; + } + + $io->table(['Table', 'Rows'], $rows); + + if (true === $dryRun) { + $io->note('Dry run: all changes have been rolled back.'); + + return Command::SUCCESS; + } + + $io->success('Anonymized the local database.'); + + return Command::SUCCESS; + } +} diff --git a/src/Service/Common/DatabaseAnonymizer.php b/src/Service/Common/DatabaseAnonymizer.php new file mode 100644 index 0000000..669995d --- /dev/null +++ b/src/Service/Common/DatabaseAnonymizer.php @@ -0,0 +1,550 @@ + + */ + private const array LOREM_IPSUM_SENTENCES = [ + 'Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.', + 'At vero eos et accusam et justo duo dolores et ea rebum.', + 'Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.', + 'Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat.', + ]; + + /** + * Blame values that are not personal data and must survive untouched. + */ + private const array PRESERVED_BLAME_VALUES = ['system']; + + private const array BLAME_COLUMNS = ['created_by', 'updated_by']; + + /** + * @var array> + */ + private array $identityMap = []; + + private int $identitySequence = 0; + + private readonly Generator $faker; + + public function __construct( + private readonly Connection $connection, + private readonly LoggerInterface $logger, + ) { + if (false === class_exists(Factory::class)) { + throw new \RuntimeException('fakerphp/faker is not installed. Run "composer require --dev fakerphp/faker".'); + } + + $this->faker = Factory::create('de_DE'); + } + + public function resetState(int $seed = self::DEFAULT_SEED): void + { + $this->identityMap = []; + $this->identitySequence = 0; + $this->faker->seed($seed); + } + + /** + * @return array rows changed, keyed by table (or table.column for blame columns) + */ + public function anonymizeAll(bool $dryRun = false, int $seed = self::DEFAULT_SEED): array + { + $this->resetState($seed); + + $this->connection->beginTransaction(); + + try { + // Order matters: each pass registers the aliases the later passes look up. + $report = [ + 'teamer' => $this->anonymizeTeamers(), + 'user' => $this->anonymizeUsers(), + 'contact' => $this->anonymizeContacts(), + 'feedback' => $this->anonymizeFeedback(), + ]; + + // The log table is deliberately left alone; it is truncated on demand instead. + $report += $this->anonymizeBlameColumns(); + + if (true === $dryRun) { + $this->connection->rollBack(); + } else { + $this->connection->commit(); + } + } catch (\Throwable $exception) { + $this->connection->rollBack(); + + throw $exception; + } + + $this->logger->info('Anonymized local database records', $report + ['dry_run' => $dryRun]); + + return $report; + } + + private function anonymizeTeamers(): int + { + $sql = <<<'SQL' + UPDATE teamer SET + first_name = :firstName, + last_name = :lastName, + date_of_birth = :dateOfBirth, + address_street = :street, + address_post_code = :postCode, + address_city = :city, + address_country = :country, + communication_email = :email, + communication_phone = :phone, + communication_mobile = :mobile, + bank_account_iban = :iban, + bank_account_bic = :bic, + bank_account_bank = :bank, + bank_account_holder = :accountHolder, + tax_id = :taxId, + health_insurance_company = :healthInsuranceCompany, + remarks = :remarks, + remarks_internal = :remarksInternal, + driver_license_review_comment = :driverLicenseReviewComment, + driver_license_reviewed_by = :driverLicenseReviewedBy + WHERE id = :id + SQL; + + $rows = $this->connection->fetchAllAssociative( + 'SELECT id, first_name, last_name, date_of_birth, communication_email,' + .' address_street, address_post_code, address_city, address_country,' + .' communication_phone, communication_mobile,' + .' bank_account_iban, bank_account_bic, bank_account_bank, bank_account_holder,' + .' tax_id, health_insurance_company, remarks, remarks_internal,' + .' driver_license_review_comment, driver_license_reviewed_by' + .' FROM teamer ORDER BY id ASC' + ); + + $changed = 0; + + foreach ($rows as $row) { + // Every teamer row is a distinct person, so never merge them by name. + $identity = $this->createIdentity(); + $this->registerAliases($identity, [ + // Keyed by id, because by the time the user pass runs the name columns + // below have already been rewritten and would no longer match. + $this->normalizeKey('teamer', (string) $row['id']), + $this->normalizeKey('email', $this->stringOrNull($row['communication_email'])), + $this->normalizeKey('name', $this->fullName($row['first_name'], $row['last_name'])), + ]); + + $changed += $this->connection->executeStatement($sql, [ + 'id' => $row['id'], + 'firstName' => $this->keepNull($row['first_name'], $identity['firstName']), + 'lastName' => $this->keepNull($row['last_name'], $identity['lastName']), + 'dateOfBirth' => $this->shiftDateOfBirth($this->stringOrNull($row['date_of_birth'])), + 'street' => $this->keepNull($row['address_street'], $identity['street']), + 'postCode' => $this->keepNull($row['address_post_code'], $identity['postCode']), + 'city' => $this->keepNull($row['address_city'], $identity['city']), + 'country' => $this->keepNull($row['address_country'], self::SYNTHETIC_COUNTRY), + 'email' => $this->keepNull($row['communication_email'], $identity['email']), + 'phone' => $this->keepNull($row['communication_phone'], $identity['phone']), + 'mobile' => $this->keepNull($row['communication_mobile'], $identity['mobile']), + 'iban' => $this->keepNull($row['bank_account_iban'], self::SYNTHETIC_IBAN), + 'bic' => $this->keepNull($row['bank_account_bic'], self::SYNTHETIC_BIC), + 'bank' => $this->keepNull($row['bank_account_bank'], $identity['bank']), + 'accountHolder' => $this->keepNull($row['bank_account_holder'], $identity['accountHolder']), + 'taxId' => $this->keepNull($row['tax_id'], $identity['taxId']), + 'healthInsuranceCompany' => $this->keepNull($row['health_insurance_company'], $identity['healthInsuranceCompany']), + 'remarks' => $this->placeholderFor($row['remarks']), + 'remarksInternal' => $this->placeholderFor($row['remarks_internal']), + 'driverLicenseReviewComment' => $this->placeholderFor($row['driver_license_review_comment']), + // Column is limited to 8 characters. + 'driverLicenseReviewedBy' => $this->keepNull($row['driver_license_reviewed_by'], $identity['initials']), + ]); + } + + return $changed; + } + + private function anonymizeUsers(): int + { + $sql = <<<'SQL' + UPDATE user SET + first_name = :firstName, + last_name = :lastName, + email = :email, + disabled_reason = :disabledReason, + disabled_reason_internal = :disabledReasonInternal + WHERE id = :id + SQL; + + $rows = $this->connection->fetchAllAssociative( + 'SELECT id, teamer_id, contact_id, first_name, last_name, email,' + .' disabled_reason, disabled_reason_internal' + .' FROM user ORDER BY id ASC' + ); + + $changed = 0; + + foreach ($rows as $row) { + // A teamer and their login must not turn into two different people. + $identity = null; + if (null !== $row['teamer_id']) { + $identity = $this->identityMap[$this->normalizeKey('teamer', (string) $row['teamer_id'])] ?? null; + } + + $identity ??= $this->createIdentity(); + + $this->registerAliases($identity, [ + $this->normalizeKey('email', $this->stringOrNull($row['email'])), + $this->normalizeKey('name', $this->fullName($row['first_name'], $row['last_name'])), + ]); + + if (null !== $row['contact_id']) { + $this->registerAliases($identity, [$this->normalizeKey('contact', (string) $row['contact_id'])]); + } + + $changed += $this->connection->executeStatement($sql, [ + 'id' => $row['id'], + 'firstName' => $this->keepNull($row['first_name'], $identity['firstName']), + 'lastName' => $this->keepNull($row['last_name'], $identity['lastName']), + 'email' => $this->keepNull($row['email'], $identity['email']), + 'disabledReason' => $this->placeholderFor($row['disabled_reason']), + 'disabledReasonInternal' => $this->placeholderFor($row['disabled_reason_internal']), + ]); + } + + return $changed; + } + + private function anonymizeContacts(): int + { + $sql = 'UPDATE contact SET name = :name, email = :email, phone = :phone WHERE id = :id'; + + $rows = $this->connection->fetchAllAssociative('SELECT id, name, email, phone FROM contact ORDER BY id ASC'); + + $changed = 0; + + foreach ($rows as $row) { + // Reuse the identity of the linked login so contact and user stay in sync. + $identity = $this->identityMap[$this->normalizeKey('contact', (string) $row['id'])] ?? null; + $identity ??= $this->createIdentity(); + + $this->registerAliases($identity, [ + $this->normalizeKey('email', $this->stringOrNull($row['email'])), + $this->normalizeKey('name', $this->stringOrNull($row['name'])), + ]); + + $changed += $this->connection->executeStatement($sql, [ + 'id' => $row['id'], + 'name' => $this->keepNull($row['name'], $identity['accountHolder']), + 'email' => $this->keepNull($row['email'], $identity['email']), + 'phone' => $this->keepNull($row['phone'], $identity['phone']), + ]); + } + + return $changed; + } + + private function anonymizeFeedback(): int + { + // author stores User::getFullName(), so it resolves against the user pass. + $authors = $this->connection->fetchFirstColumn( + "SELECT DISTINCT author FROM feedback WHERE author IS NOT NULL AND author <> ''" + ); + + $changed = 0; + + foreach ($authors as $author) { + $identity = $this->lookupIdentity($author) ?? $this->identityForUnknown($author); + + $changed += $this->connection->executeStatement( + 'UPDATE feedback SET author = :new WHERE author = :old', + ['new' => $identity['accountHolder'], 'old' => $author] + ); + } + + $comments = $this->connection->fetchAllAssociative( + 'SELECT id, comment, comment_internal FROM feedback' + ." WHERE (comment IS NOT NULL AND comment <> '')" + ." OR (comment_internal IS NOT NULL AND comment_internal <> '')" + ); + + foreach ($comments as $row) { + $this->connection->executeStatement( + 'UPDATE feedback SET comment = :comment, comment_internal = :commentInternal WHERE id = :id', + [ + 'id' => $row['id'], + 'comment' => $this->placeholderFor($row['comment']), + 'commentInternal' => $this->placeholderFor($row['comment_internal']), + ] + ); + } + + return $changed; + } + + /** + * Rewrites created_by/updated_by wherever they exist. Columns are discovered from + * the schema so tables adopting the blameable trait later are covered automatically. + * + * @return array + */ + private function anonymizeBlameColumns(): array + { + $report = []; + + foreach ($this->findBlameColumns() as [$table, $column]) { + $quotedTable = $this->connection->quoteIdentifier($table); + $quotedColumn = $this->connection->quoteIdentifier($column); + + $values = $this->connection->fetchFirstColumn( + sprintf('SELECT DISTINCT %s FROM %s WHERE %s IS NOT NULL', $quotedColumn, $quotedTable, $quotedColumn) + ); + + $changed = 0; + + foreach ($values as $value) { + $original = $this->stringOrNull($value); + + if (null === $original || true === in_array(mb_strtolower($original), self::PRESERVED_BLAME_VALUES, true)) { + continue; + } + + // Unmatched values are former staff or importers; each keeps its own identity. + $identity = $this->lookupIdentity($original) ?? $this->identityForUnknown($original); + + $changed += $this->connection->executeStatement( + sprintf('UPDATE %s SET %s = :new WHERE %s = :old', $quotedTable, $quotedColumn, $quotedColumn), + ['new' => $identity['email'], 'old' => $value] + ); + } + + if ($changed > 0) { + $report[$table.'.'.$column] = $changed; + } + } + + return $report; + } + + /** + * @return list + */ + private function findBlameColumns(): array + { + $schemaManager = $this->connection->createSchemaManager(); + $columns = []; + + foreach ($schemaManager->listTableNames() as $table) { + $tableColumns = array_map( + static fn ($column) => $column->getName(), + $schemaManager->listTableColumns($table) + ); + + foreach (self::BLAME_COLUMNS as $blameColumn) { + if (true === in_array($blameColumn, $tableColumns, true)) { + $columns[] = [$table, $blameColumn]; + } + } + } + + return $columns; + } + + /** + * @return array + */ + private function identityForUnknown(string $value): array + { + $identity = $this->createIdentity(); + + $this->registerAliases($identity, [ + $this->normalizeKey($this->looksLikeEmail($value) ? 'email' : 'name', $value), + ]); + + return $identity; + } + + /** + * @return array|null + */ + private function lookupIdentity(?string $value): ?array + { + $normalized = $this->stringOrNull($value); + + if (null === $normalized) { + return null; + } + + return $this->identityMap[$this->normalizeKey('email', $normalized)] + ?? $this->identityMap[$this->normalizeKey('name', $normalized)] + ?? null; + } + + /** + * Aliases are never reassigned: two different people sharing a name must not + * collapse into a single identity. + * + * @param array $identity + * @param list $aliases + */ + private function registerAliases(array $identity, array $aliases): void + { + foreach ($aliases as $alias) { + if (null === $alias || true === isset($this->identityMap[$alias])) { + continue; + } + + $this->identityMap[$alias] = $identity; + } + } + + /** + * @return array + */ + private function createIdentity(): array + { + ++$this->identitySequence; + $sequence = $this->identitySequence; + + $firstName = $this->faker->firstName(); + $lastName = $this->faker->lastName(); + $emailLocal = sprintf('%s.%s.%04d', $this->slug($firstName), $this->slug($lastName), $sequence); + + return [ + 'firstName' => $firstName, + 'lastName' => $lastName, + 'accountHolder' => $firstName.' '.$lastName, + 'initials' => mb_substr(mb_strtoupper(mb_substr($firstName, 0, 1).mb_substr($lastName, 0, 2)), 0, 8), + 'email' => sprintf('%s@%s', $emailLocal, self::SYNTHETIC_EMAIL_DOMAIN), + 'mobile' => $this->faker->numerify('01#########'), + 'phone' => $this->faker->numerify('0##########'), + 'street' => sprintf('%s %s', $this->faker->streetName(), $this->faker->buildingNumber()), + 'postCode' => $this->faker->postcode(), + 'city' => $this->faker->city(), + 'bank' => $this->faker->company(), + 'taxId' => $this->faker->numerify('###########'), + 'healthInsuranceCompany' => $this->faker->company(), + ]; + } + + private function shiftDateOfBirth(?string $dateOfBirth): ?string + { + if (null === $dateOfBirth) { + return null; + } + + $date = \DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $dateOfBirth) + ?: \DateTimeImmutable::createFromFormat('Y-m-d', substr($dateOfBirth, 0, 10)); + + if (false === $date) { + return null; + } + + // Keep the birth year so age-based logic and statistics stay meaningful. + return sprintf('%s-%02d-%02d', $date->format('Y'), $this->faker->numberBetween(1, 12), $this->faker->numberBetween(1, 28)); + } + + private function placeholderFor(?string $value): ?string + { + if (null === $this->stringOrNull($value)) { + return $value; + } + + $sentences = array_slice( + self::LOREM_IPSUM_SENTENCES, + 0, + $this->faker->numberBetween(1, count(self::LOREM_IPSUM_SENTENCES)) + ); + + return implode(' ', $sentences); + } + + /** + * Empty columns stay empty so "has a value" remains visible in the anonymized data. + */ + private function keepNull(?string $value, string $replacement): ?string + { + return null === $this->stringOrNull($value) ? $value : $replacement; + } + + private function fullName(?string $firstName, ?string $lastName): ?string + { + return $this->stringOrNull(trim(($firstName ?? '').' '.($lastName ?? ''))); + } + + private function looksLikeEmail(string $value): bool + { + return false !== filter_var($value, \FILTER_VALIDATE_EMAIL); + } + + private function normalizeKey(string $type, ?string $value): ?string + { + $normalized = $this->stringOrNull($value); + + if (null === $normalized) { + return null; + } + + return $type.':'.mb_strtolower($normalized); + } + + private function stringOrNull(mixed $value): ?string + { + if (null === $value) { + return null; + } + + $trimmed = trim((string) $value); + + return '' === $trimmed ? null : $trimmed; + } + + private function slug(string $value): string + { + $normalizedValue = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $value); + if (false === $normalizedValue) { + $normalizedValue = $value; + } + + $normalizedValue = strtolower($normalizedValue); + $normalizedValue = preg_replace('/[^a-z0-9]+/', '.', $normalizedValue); + $normalizedValue = trim((string) $normalizedValue, '.'); + + if ('' === $normalizedValue) { + return 'user'; + } + + return $normalizedValue; + } +}