Handle records as entities
This commit is contained in:
parent
1e275422bf
commit
ec7f72cbe4
|
@ -95,6 +95,13 @@ class CsvImportCommand extends Command
|
||||||
'Name of the CSV column which holds the records\' sets list.',
|
'Name of the CSV column which holds the records\' sets list.',
|
||||||
'sets'
|
'sets'
|
||||||
);
|
);
|
||||||
|
$this->addOption(
|
||||||
|
'noValidation',
|
||||||
|
null,
|
||||||
|
InputOption::VALUE_NONE,
|
||||||
|
'Omit content validation (improves performance for large record sets).',
|
||||||
|
false
|
||||||
|
);
|
||||||
parent::configure();
|
parent::configure();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -117,6 +124,8 @@ class CsvImportCommand extends Command
|
||||||
$arguments = $input->getArguments();
|
$arguments = $input->getArguments();
|
||||||
/** @var Format */
|
/** @var Format */
|
||||||
$format = Database::getInstance()->getEntityManager()->getReference(Format::class, $arguments['format']);
|
$format = Database::getInstance()->getEntityManager()->getReference(Format::class, $arguments['format']);
|
||||||
|
/** @var bool */
|
||||||
|
$noValidation = $input->getOption('noValidation');
|
||||||
/** @var resource */
|
/** @var resource */
|
||||||
$file = fopen($arguments['file'], 'r');
|
$file = fopen($arguments['file'], 'r');
|
||||||
|
|
||||||
|
@ -129,23 +138,25 @@ class CsvImportCommand extends Command
|
||||||
$progressIndicator = new ProgressIndicator($output, 'verbose', 200, ['⠏', '⠛', '⠹', '⢸', '⣰', '⣤', '⣆', '⡇']);
|
$progressIndicator = new ProgressIndicator($output, 'verbose', 200, ['⠏', '⠛', '⠹', '⢸', '⣰', '⣤', '⣆', '⡇']);
|
||||||
$progressIndicator->start('Importing...');
|
$progressIndicator->start('Importing...');
|
||||||
|
|
||||||
while ($record = fgetcsv($file)) {
|
while ($row = fgetcsv($file)) {
|
||||||
Database::getInstance()->addOrUpdateRecord(
|
$record = new Record(
|
||||||
$record[$columns['idColumn']],
|
$row[$columns['idColumn']],
|
||||||
$format,
|
$format,
|
||||||
trim($record[$columns['contentColumn']]),
|
null,
|
||||||
new DateTime($record[$columns['dateColumn']] ?? 'now'),
|
new DateTime($row[$columns['dateColumn']] ?? 'now')
|
||||||
// TODO: Complete support for sets.
|
|
||||||
/* $record[$columns['setColumn']] ?? */ null,
|
|
||||||
true
|
|
||||||
);
|
);
|
||||||
|
if (strlen(trim($row[$columns['contentColumn']])) > 0) {
|
||||||
|
$record->setContent($row[$columns['contentColumn']], !$noValidation);
|
||||||
|
}
|
||||||
|
// TODO: Complete support for sets.
|
||||||
|
Database::getInstance()->addOrUpdateRecord($record, true);
|
||||||
|
|
||||||
++$count;
|
++$count;
|
||||||
$progressIndicator->advance();
|
$progressIndicator->advance();
|
||||||
$progressIndicator->setMessage((string) $count . ' done.');
|
$progressIndicator->setMessage((string) $count . ' done.');
|
||||||
|
|
||||||
// Flush to database if memory usage reaches 90% of available limit.
|
// Flush to database if memory usage reaches 90% of available limit.
|
||||||
if (memory_get_usage() / $memoryLimit > 0.9) {
|
if ((memory_get_usage() / $memoryLimit) > 0.9) {
|
||||||
Database::getInstance()->flush([Record::class]);
|
Database::getInstance()->flush([Record::class]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -92,46 +92,32 @@ class Database
|
||||||
/**
|
/**
|
||||||
* Add or update record.
|
* Add or update record.
|
||||||
*
|
*
|
||||||
* @param string $identifier The record identifier
|
* @param Record $newRecord The record
|
||||||
* @param Format $format The metadata prefix
|
|
||||||
* @param ?string $data The record's content
|
|
||||||
* @param ?DateTime $lastChanged The date of last change
|
|
||||||
* @param ?array<string, Set> $sets The record's associated sets
|
|
||||||
* @param bool $bulkMode Should we operate in bulk mode (no flush)?
|
* @param bool $bulkMode Should we operate in bulk mode (no flush)?
|
||||||
*
|
*
|
||||||
* @return void
|
* @return void
|
||||||
*/
|
*/
|
||||||
public function addOrUpdateRecord(
|
public function addOrUpdateRecord(Record $newRecord, bool $bulkMode = false): void
|
||||||
string $identifier,
|
|
||||||
Format $format,
|
|
||||||
?string $data = null,
|
|
||||||
?DateTime $lastChanged = null,
|
|
||||||
// TODO: Complete support for sets
|
|
||||||
?array $sets,
|
|
||||||
bool $bulkMode = false
|
|
||||||
): void
|
|
||||||
{
|
{
|
||||||
$record = $this->entityManager->find(Record::class, ['identifier' => $identifier, 'format' => $format]);
|
$oldRecord = $this->entityManager->find(
|
||||||
if (!isset($data) && Configuration::getInstance()->deletedRecords === 'no') {
|
Record::class,
|
||||||
if (isset($record)) {
|
[
|
||||||
$this->entityManager->remove($record);
|
'identifier' => $newRecord->getIdentifier(),
|
||||||
|
'format' => $newRecord->getFormat()
|
||||||
|
]
|
||||||
|
);
|
||||||
|
if (isset($oldRecord)) {
|
||||||
|
if ($newRecord->hasContent() || Configuration::getInstance()->deletedRecords !== 'no') {
|
||||||
|
$oldRecord->setContent($newRecord->getContent(), false);
|
||||||
|
$oldRecord->setLastChanged($newRecord->getLastChanged());
|
||||||
|
// TODO: Add full set support.
|
||||||
|
} else {
|
||||||
|
$this->entityManager->remove($oldRecord);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (isset($record)) {
|
if ($newRecord->hasContent() || Configuration::getInstance()->deletedRecords !== 'no') {
|
||||||
try {
|
$this->entityManager->persist($newRecord);
|
||||||
$record->setContent($data);
|
|
||||||
$record->setLastChanged($lastChanged);
|
|
||||||
} catch (ValidationFailedException $exception) {
|
|
||||||
throw $exception;
|
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
try {
|
|
||||||
$record = new Record($identifier, $format, $data, $lastChanged);
|
|
||||||
} catch (ValidationFailedException $exception) {
|
|
||||||
throw $exception;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
$this->entityManager->persist($record);
|
|
||||||
}
|
}
|
||||||
if (!$bulkMode) {
|
if (!$bulkMode) {
|
||||||
$this->entityManager->flush();
|
$this->entityManager->flush();
|
||||||
|
|
|
@ -156,6 +156,16 @@ class Record
|
||||||
return $this->sets->toArray();
|
return $this->sets->toArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Whether this record has any content.
|
||||||
|
*
|
||||||
|
* @return bool TRUE if content exists, FALSE otherwise
|
||||||
|
*/
|
||||||
|
public function hasContent(): bool
|
||||||
|
{
|
||||||
|
return isset($this->content);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Remove record from set.
|
* Remove record from set.
|
||||||
*
|
*
|
||||||
|
@ -184,6 +194,7 @@ class Record
|
||||||
public function setContent(?string $data = null, bool $validate = true): void
|
public function setContent(?string $data = null, bool $validate = true): void
|
||||||
{
|
{
|
||||||
if (isset($data)) {
|
if (isset($data)) {
|
||||||
|
$data = trim($data);
|
||||||
if ($validate) {
|
if ($validate) {
|
||||||
try {
|
try {
|
||||||
$data = $this->validate($data);
|
$data = $this->validate($data);
|
||||||
|
@ -233,7 +244,6 @@ class Record
|
||||||
*/
|
*/
|
||||||
protected function validate(string $xml): string
|
protected function validate(string $xml): string
|
||||||
{
|
{
|
||||||
$xml = trim($xml);
|
|
||||||
$validator = Validation::createValidator();
|
$validator = Validation::createValidator();
|
||||||
$violations = $validator->validate(
|
$violations = $validator->validate(
|
||||||
$xml,
|
$xml,
|
||||||
|
|
Loading…
Reference in New Issue