feat: database anonymization command
This commit is contained in:
Björn Fromme
@@ -0,0 +1,550 @@
|
||||
<?php
|
||||
|
||||
namespace App\Service\Common;
|
||||
|
||||
use Doctrine\DBAL\Connection;
|
||||
use Faker\Factory;
|
||||
use Faker\Generator;
|
||||
use Psr\Log\LoggerInterface;
|
||||
|
||||
/**
|
||||
* Replaces personal data in the local database with synthetic values.
|
||||
*
|
||||
* Works on the DBAL layer on purpose: the ORM would trigger the blameable and
|
||||
* timestampable listeners on every update, overwriting the very created_by and
|
||||
* updated_by values that have to be rewritten consistently here, and bumping
|
||||
* updated_at on every touched row.
|
||||
*
|
||||
* A person can appear as an email in created_by/updated_by, as a full name in
|
||||
* feedback.author, and as separate columns in teamer, user and contact. All of
|
||||
* these resolve to the same synthetic identity so the data stays coherent to work
|
||||
* with after anonymization.
|
||||
*
|
||||
* The log table is out of scope even though it holds personal data in its JSON
|
||||
* payloads; it is truncated on demand instead.
|
||||
*/
|
||||
class DatabaseAnonymizer
|
||||
{
|
||||
private const string SYNTHETIC_EMAIL_DOMAIN = 'example.test';
|
||||
private const string SYNTHETIC_COUNTRY = 'DE';
|
||||
private const string SYNTHETIC_IBAN = 'DE89370400440532013000';
|
||||
private const string SYNTHETIC_BIC = 'GENODEF1SYN';
|
||||
private const int DEFAULT_SEED = 20240101;
|
||||
|
||||
/**
|
||||
* Free text is replaced with classical lorem ipsum: instantly recognizable as
|
||||
* placeholder and, unlike Faker's German text provider, not distracting to read
|
||||
* for German speakers working with the anonymized data.
|
||||
*
|
||||
* @var list<string>
|
||||
*/
|
||||
private const array LOREM_IPSUM_SENTENCES = [
|
||||
'Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua.',
|
||||
'At vero eos et accusam et justo duo dolores et ea rebum.',
|
||||
'Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.',
|
||||
'Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat.',
|
||||
];
|
||||
|
||||
/**
|
||||
* Blame values that are not personal data and must survive untouched.
|
||||
*/
|
||||
private const array PRESERVED_BLAME_VALUES = ['system'];
|
||||
|
||||
private const array BLAME_COLUMNS = ['created_by', 'updated_by'];
|
||||
|
||||
/**
|
||||
* @var array<string, array<string, string>>
|
||||
*/
|
||||
private array $identityMap = [];
|
||||
|
||||
private int $identitySequence = 0;
|
||||
|
||||
private readonly Generator $faker;
|
||||
|
||||
public function __construct(
|
||||
private readonly Connection $connection,
|
||||
private readonly LoggerInterface $logger,
|
||||
) {
|
||||
if (false === class_exists(Factory::class)) {
|
||||
throw new \RuntimeException('fakerphp/faker is not installed. Run "composer require --dev fakerphp/faker".');
|
||||
}
|
||||
|
||||
$this->faker = Factory::create('de_DE');
|
||||
}
|
||||
|
||||
public function resetState(int $seed = self::DEFAULT_SEED): void
|
||||
{
|
||||
$this->identityMap = [];
|
||||
$this->identitySequence = 0;
|
||||
$this->faker->seed($seed);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, int> rows changed, keyed by table (or table.column for blame columns)
|
||||
*/
|
||||
public function anonymizeAll(bool $dryRun = false, int $seed = self::DEFAULT_SEED): array
|
||||
{
|
||||
$this->resetState($seed);
|
||||
|
||||
$this->connection->beginTransaction();
|
||||
|
||||
try {
|
||||
// Order matters: each pass registers the aliases the later passes look up.
|
||||
$report = [
|
||||
'teamer' => $this->anonymizeTeamers(),
|
||||
'user' => $this->anonymizeUsers(),
|
||||
'contact' => $this->anonymizeContacts(),
|
||||
'feedback' => $this->anonymizeFeedback(),
|
||||
];
|
||||
|
||||
// The log table is deliberately left alone; it is truncated on demand instead.
|
||||
$report += $this->anonymizeBlameColumns();
|
||||
|
||||
if (true === $dryRun) {
|
||||
$this->connection->rollBack();
|
||||
} else {
|
||||
$this->connection->commit();
|
||||
}
|
||||
} catch (\Throwable $exception) {
|
||||
$this->connection->rollBack();
|
||||
|
||||
throw $exception;
|
||||
}
|
||||
|
||||
$this->logger->info('Anonymized local database records', $report + ['dry_run' => $dryRun]);
|
||||
|
||||
return $report;
|
||||
}
|
||||
|
||||
private function anonymizeTeamers(): int
|
||||
{
|
||||
$sql = <<<'SQL'
|
||||
UPDATE teamer SET
|
||||
first_name = :firstName,
|
||||
last_name = :lastName,
|
||||
date_of_birth = :dateOfBirth,
|
||||
address_street = :street,
|
||||
address_post_code = :postCode,
|
||||
address_city = :city,
|
||||
address_country = :country,
|
||||
communication_email = :email,
|
||||
communication_phone = :phone,
|
||||
communication_mobile = :mobile,
|
||||
bank_account_iban = :iban,
|
||||
bank_account_bic = :bic,
|
||||
bank_account_bank = :bank,
|
||||
bank_account_holder = :accountHolder,
|
||||
tax_id = :taxId,
|
||||
health_insurance_company = :healthInsuranceCompany,
|
||||
remarks = :remarks,
|
||||
remarks_internal = :remarksInternal,
|
||||
driver_license_review_comment = :driverLicenseReviewComment,
|
||||
driver_license_reviewed_by = :driverLicenseReviewedBy
|
||||
WHERE id = :id
|
||||
SQL;
|
||||
|
||||
$rows = $this->connection->fetchAllAssociative(
|
||||
'SELECT id, first_name, last_name, date_of_birth, communication_email,'
|
||||
.' address_street, address_post_code, address_city, address_country,'
|
||||
.' communication_phone, communication_mobile,'
|
||||
.' bank_account_iban, bank_account_bic, bank_account_bank, bank_account_holder,'
|
||||
.' tax_id, health_insurance_company, remarks, remarks_internal,'
|
||||
.' driver_license_review_comment, driver_license_reviewed_by'
|
||||
.' FROM teamer ORDER BY id ASC'
|
||||
);
|
||||
|
||||
$changed = 0;
|
||||
|
||||
foreach ($rows as $row) {
|
||||
// Every teamer row is a distinct person, so never merge them by name.
|
||||
$identity = $this->createIdentity();
|
||||
$this->registerAliases($identity, [
|
||||
// Keyed by id, because by the time the user pass runs the name columns
|
||||
// below have already been rewritten and would no longer match.
|
||||
$this->normalizeKey('teamer', (string) $row['id']),
|
||||
$this->normalizeKey('email', $this->stringOrNull($row['communication_email'])),
|
||||
$this->normalizeKey('name', $this->fullName($row['first_name'], $row['last_name'])),
|
||||
]);
|
||||
|
||||
$changed += $this->connection->executeStatement($sql, [
|
||||
'id' => $row['id'],
|
||||
'firstName' => $this->keepNull($row['first_name'], $identity['firstName']),
|
||||
'lastName' => $this->keepNull($row['last_name'], $identity['lastName']),
|
||||
'dateOfBirth' => $this->shiftDateOfBirth($this->stringOrNull($row['date_of_birth'])),
|
||||
'street' => $this->keepNull($row['address_street'], $identity['street']),
|
||||
'postCode' => $this->keepNull($row['address_post_code'], $identity['postCode']),
|
||||
'city' => $this->keepNull($row['address_city'], $identity['city']),
|
||||
'country' => $this->keepNull($row['address_country'], self::SYNTHETIC_COUNTRY),
|
||||
'email' => $this->keepNull($row['communication_email'], $identity['email']),
|
||||
'phone' => $this->keepNull($row['communication_phone'], $identity['phone']),
|
||||
'mobile' => $this->keepNull($row['communication_mobile'], $identity['mobile']),
|
||||
'iban' => $this->keepNull($row['bank_account_iban'], self::SYNTHETIC_IBAN),
|
||||
'bic' => $this->keepNull($row['bank_account_bic'], self::SYNTHETIC_BIC),
|
||||
'bank' => $this->keepNull($row['bank_account_bank'], $identity['bank']),
|
||||
'accountHolder' => $this->keepNull($row['bank_account_holder'], $identity['accountHolder']),
|
||||
'taxId' => $this->keepNull($row['tax_id'], $identity['taxId']),
|
||||
'healthInsuranceCompany' => $this->keepNull($row['health_insurance_company'], $identity['healthInsuranceCompany']),
|
||||
'remarks' => $this->placeholderFor($row['remarks']),
|
||||
'remarksInternal' => $this->placeholderFor($row['remarks_internal']),
|
||||
'driverLicenseReviewComment' => $this->placeholderFor($row['driver_license_review_comment']),
|
||||
// Column is limited to 8 characters.
|
||||
'driverLicenseReviewedBy' => $this->keepNull($row['driver_license_reviewed_by'], $identity['initials']),
|
||||
]);
|
||||
}
|
||||
|
||||
return $changed;
|
||||
}
|
||||
|
||||
private function anonymizeUsers(): int
|
||||
{
|
||||
$sql = <<<'SQL'
|
||||
UPDATE user SET
|
||||
first_name = :firstName,
|
||||
last_name = :lastName,
|
||||
email = :email,
|
||||
disabled_reason = :disabledReason,
|
||||
disabled_reason_internal = :disabledReasonInternal
|
||||
WHERE id = :id
|
||||
SQL;
|
||||
|
||||
$rows = $this->connection->fetchAllAssociative(
|
||||
'SELECT id, teamer_id, contact_id, first_name, last_name, email,'
|
||||
.' disabled_reason, disabled_reason_internal'
|
||||
.' FROM user ORDER BY id ASC'
|
||||
);
|
||||
|
||||
$changed = 0;
|
||||
|
||||
foreach ($rows as $row) {
|
||||
// A teamer and their login must not turn into two different people.
|
||||
$identity = null;
|
||||
if (null !== $row['teamer_id']) {
|
||||
$identity = $this->identityMap[$this->normalizeKey('teamer', (string) $row['teamer_id'])] ?? null;
|
||||
}
|
||||
|
||||
$identity ??= $this->createIdentity();
|
||||
|
||||
$this->registerAliases($identity, [
|
||||
$this->normalizeKey('email', $this->stringOrNull($row['email'])),
|
||||
$this->normalizeKey('name', $this->fullName($row['first_name'], $row['last_name'])),
|
||||
]);
|
||||
|
||||
if (null !== $row['contact_id']) {
|
||||
$this->registerAliases($identity, [$this->normalizeKey('contact', (string) $row['contact_id'])]);
|
||||
}
|
||||
|
||||
$changed += $this->connection->executeStatement($sql, [
|
||||
'id' => $row['id'],
|
||||
'firstName' => $this->keepNull($row['first_name'], $identity['firstName']),
|
||||
'lastName' => $this->keepNull($row['last_name'], $identity['lastName']),
|
||||
'email' => $this->keepNull($row['email'], $identity['email']),
|
||||
'disabledReason' => $this->placeholderFor($row['disabled_reason']),
|
||||
'disabledReasonInternal' => $this->placeholderFor($row['disabled_reason_internal']),
|
||||
]);
|
||||
}
|
||||
|
||||
return $changed;
|
||||
}
|
||||
|
||||
private function anonymizeContacts(): int
|
||||
{
|
||||
$sql = 'UPDATE contact SET name = :name, email = :email, phone = :phone WHERE id = :id';
|
||||
|
||||
$rows = $this->connection->fetchAllAssociative('SELECT id, name, email, phone FROM contact ORDER BY id ASC');
|
||||
|
||||
$changed = 0;
|
||||
|
||||
foreach ($rows as $row) {
|
||||
// Reuse the identity of the linked login so contact and user stay in sync.
|
||||
$identity = $this->identityMap[$this->normalizeKey('contact', (string) $row['id'])] ?? null;
|
||||
$identity ??= $this->createIdentity();
|
||||
|
||||
$this->registerAliases($identity, [
|
||||
$this->normalizeKey('email', $this->stringOrNull($row['email'])),
|
||||
$this->normalizeKey('name', $this->stringOrNull($row['name'])),
|
||||
]);
|
||||
|
||||
$changed += $this->connection->executeStatement($sql, [
|
||||
'id' => $row['id'],
|
||||
'name' => $this->keepNull($row['name'], $identity['accountHolder']),
|
||||
'email' => $this->keepNull($row['email'], $identity['email']),
|
||||
'phone' => $this->keepNull($row['phone'], $identity['phone']),
|
||||
]);
|
||||
}
|
||||
|
||||
return $changed;
|
||||
}
|
||||
|
||||
private function anonymizeFeedback(): int
|
||||
{
|
||||
// author stores User::getFullName(), so it resolves against the user pass.
|
||||
$authors = $this->connection->fetchFirstColumn(
|
||||
"SELECT DISTINCT author FROM feedback WHERE author IS NOT NULL AND author <> ''"
|
||||
);
|
||||
|
||||
$changed = 0;
|
||||
|
||||
foreach ($authors as $author) {
|
||||
$identity = $this->lookupIdentity($author) ?? $this->identityForUnknown($author);
|
||||
|
||||
$changed += $this->connection->executeStatement(
|
||||
'UPDATE feedback SET author = :new WHERE author = :old',
|
||||
['new' => $identity['accountHolder'], 'old' => $author]
|
||||
);
|
||||
}
|
||||
|
||||
$comments = $this->connection->fetchAllAssociative(
|
||||
'SELECT id, comment, comment_internal FROM feedback'
|
||||
." WHERE (comment IS NOT NULL AND comment <> '')"
|
||||
." OR (comment_internal IS NOT NULL AND comment_internal <> '')"
|
||||
);
|
||||
|
||||
foreach ($comments as $row) {
|
||||
$this->connection->executeStatement(
|
||||
'UPDATE feedback SET comment = :comment, comment_internal = :commentInternal WHERE id = :id',
|
||||
[
|
||||
'id' => $row['id'],
|
||||
'comment' => $this->placeholderFor($row['comment']),
|
||||
'commentInternal' => $this->placeholderFor($row['comment_internal']),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
return $changed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrites created_by/updated_by wherever they exist. Columns are discovered from
|
||||
* the schema so tables adopting the blameable trait later are covered automatically.
|
||||
*
|
||||
* @return array<string, int>
|
||||
*/
|
||||
private function anonymizeBlameColumns(): array
|
||||
{
|
||||
$report = [];
|
||||
|
||||
foreach ($this->findBlameColumns() as [$table, $column]) {
|
||||
$quotedTable = $this->connection->quoteIdentifier($table);
|
||||
$quotedColumn = $this->connection->quoteIdentifier($column);
|
||||
|
||||
$values = $this->connection->fetchFirstColumn(
|
||||
sprintf('SELECT DISTINCT %s FROM %s WHERE %s IS NOT NULL', $quotedColumn, $quotedTable, $quotedColumn)
|
||||
);
|
||||
|
||||
$changed = 0;
|
||||
|
||||
foreach ($values as $value) {
|
||||
$original = $this->stringOrNull($value);
|
||||
|
||||
if (null === $original || true === in_array(mb_strtolower($original), self::PRESERVED_BLAME_VALUES, true)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Unmatched values are former staff or importers; each keeps its own identity.
|
||||
$identity = $this->lookupIdentity($original) ?? $this->identityForUnknown($original);
|
||||
|
||||
$changed += $this->connection->executeStatement(
|
||||
sprintf('UPDATE %s SET %s = :new WHERE %s = :old', $quotedTable, $quotedColumn, $quotedColumn),
|
||||
['new' => $identity['email'], 'old' => $value]
|
||||
);
|
||||
}
|
||||
|
||||
if ($changed > 0) {
|
||||
$report[$table.'.'.$column] = $changed;
|
||||
}
|
||||
}
|
||||
|
||||
return $report;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return list<array{0: string, 1: string}>
|
||||
*/
|
||||
private function findBlameColumns(): array
|
||||
{
|
||||
$schemaManager = $this->connection->createSchemaManager();
|
||||
$columns = [];
|
||||
|
||||
foreach ($schemaManager->listTableNames() as $table) {
|
||||
$tableColumns = array_map(
|
||||
static fn ($column) => $column->getName(),
|
||||
$schemaManager->listTableColumns($table)
|
||||
);
|
||||
|
||||
foreach (self::BLAME_COLUMNS as $blameColumn) {
|
||||
if (true === in_array($blameColumn, $tableColumns, true)) {
|
||||
$columns[] = [$table, $blameColumn];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $columns;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, string>
|
||||
*/
|
||||
private function identityForUnknown(string $value): array
|
||||
{
|
||||
$identity = $this->createIdentity();
|
||||
|
||||
$this->registerAliases($identity, [
|
||||
$this->normalizeKey($this->looksLikeEmail($value) ? 'email' : 'name', $value),
|
||||
]);
|
||||
|
||||
return $identity;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, string>|null
|
||||
*/
|
||||
private function lookupIdentity(?string $value): ?array
|
||||
{
|
||||
$normalized = $this->stringOrNull($value);
|
||||
|
||||
if (null === $normalized) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $this->identityMap[$this->normalizeKey('email', $normalized)]
|
||||
?? $this->identityMap[$this->normalizeKey('name', $normalized)]
|
||||
?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Aliases are never reassigned: two different people sharing a name must not
|
||||
* collapse into a single identity.
|
||||
*
|
||||
* @param array<string, string> $identity
|
||||
* @param list<string|null> $aliases
|
||||
*/
|
||||
private function registerAliases(array $identity, array $aliases): void
|
||||
{
|
||||
foreach ($aliases as $alias) {
|
||||
if (null === $alias || true === isset($this->identityMap[$alias])) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->identityMap[$alias] = $identity;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array<string, string>
|
||||
*/
|
||||
private function createIdentity(): array
|
||||
{
|
||||
++$this->identitySequence;
|
||||
$sequence = $this->identitySequence;
|
||||
|
||||
$firstName = $this->faker->firstName();
|
||||
$lastName = $this->faker->lastName();
|
||||
$emailLocal = sprintf('%s.%s.%04d', $this->slug($firstName), $this->slug($lastName), $sequence);
|
||||
|
||||
return [
|
||||
'firstName' => $firstName,
|
||||
'lastName' => $lastName,
|
||||
'accountHolder' => $firstName.' '.$lastName,
|
||||
'initials' => mb_substr(mb_strtoupper(mb_substr($firstName, 0, 1).mb_substr($lastName, 0, 2)), 0, 8),
|
||||
'email' => sprintf('%s@%s', $emailLocal, self::SYNTHETIC_EMAIL_DOMAIN),
|
||||
'mobile' => $this->faker->numerify('01#########'),
|
||||
'phone' => $this->faker->numerify('0##########'),
|
||||
'street' => sprintf('%s %s', $this->faker->streetName(), $this->faker->buildingNumber()),
|
||||
'postCode' => $this->faker->postcode(),
|
||||
'city' => $this->faker->city(),
|
||||
'bank' => $this->faker->company(),
|
||||
'taxId' => $this->faker->numerify('###########'),
|
||||
'healthInsuranceCompany' => $this->faker->company(),
|
||||
];
|
||||
}
|
||||
|
||||
private function shiftDateOfBirth(?string $dateOfBirth): ?string
|
||||
{
|
||||
if (null === $dateOfBirth) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$date = \DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $dateOfBirth)
|
||||
?: \DateTimeImmutable::createFromFormat('Y-m-d', substr($dateOfBirth, 0, 10));
|
||||
|
||||
if (false === $date) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Keep the birth year so age-based logic and statistics stay meaningful.
|
||||
return sprintf('%s-%02d-%02d', $date->format('Y'), $this->faker->numberBetween(1, 12), $this->faker->numberBetween(1, 28));
|
||||
}
|
||||
|
||||
private function placeholderFor(?string $value): ?string
|
||||
{
|
||||
if (null === $this->stringOrNull($value)) {
|
||||
return $value;
|
||||
}
|
||||
|
||||
$sentences = array_slice(
|
||||
self::LOREM_IPSUM_SENTENCES,
|
||||
0,
|
||||
$this->faker->numberBetween(1, count(self::LOREM_IPSUM_SENTENCES))
|
||||
);
|
||||
|
||||
return implode(' ', $sentences);
|
||||
}
|
||||
|
||||
/**
|
||||
* Empty columns stay empty so "has a value" remains visible in the anonymized data.
|
||||
*/
|
||||
private function keepNull(?string $value, string $replacement): ?string
|
||||
{
|
||||
return null === $this->stringOrNull($value) ? $value : $replacement;
|
||||
}
|
||||
|
||||
private function fullName(?string $firstName, ?string $lastName): ?string
|
||||
{
|
||||
return $this->stringOrNull(trim(($firstName ?? '').' '.($lastName ?? '')));
|
||||
}
|
||||
|
||||
private function looksLikeEmail(string $value): bool
|
||||
{
|
||||
return false !== filter_var($value, \FILTER_VALIDATE_EMAIL);
|
||||
}
|
||||
|
||||
private function normalizeKey(string $type, ?string $value): ?string
|
||||
{
|
||||
$normalized = $this->stringOrNull($value);
|
||||
|
||||
if (null === $normalized) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return $type.':'.mb_strtolower($normalized);
|
||||
}
|
||||
|
||||
private function stringOrNull(mixed $value): ?string
|
||||
{
|
||||
if (null === $value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
$trimmed = trim((string) $value);
|
||||
|
||||
return '' === $trimmed ? null : $trimmed;
|
||||
}
|
||||
|
||||
private function slug(string $value): string
|
||||
{
|
||||
$normalizedValue = iconv('UTF-8', 'ASCII//TRANSLIT//IGNORE', $value);
|
||||
if (false === $normalizedValue) {
|
||||
$normalizedValue = $value;
|
||||
}
|
||||
|
||||
$normalizedValue = strtolower($normalizedValue);
|
||||
$normalizedValue = preg_replace('/[^a-z0-9]+/', '.', $normalizedValue);
|
||||
$normalizedValue = trim((string) $normalizedValue, '.');
|
||||
|
||||
if ('' === $normalizedValue) {
|
||||
return 'user';
|
||||
}
|
||||
|
||||
return $normalizedValue;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user