Make memory limit configurable

config/config.dist.yml

@@ -16,7 +16,7 @@ repositoryName: 'OAI-PMH 2.0 Data Provider'
 #
 # This has to be a valid email according to RFC 822 Address Specification.
 #
-adminEmail: admin@example.org
+adminEmail: 'admin@example.org'
 
 #
 # Database connection details
@@ -41,13 +41,27 @@ adminEmail: admin@example.org
 # database: 'mssql://oaipmh:secret@127.0.0.1/oaipmh'
 # database: 'mysql://root@localhost/oai?charset=utf8mb4'
 # database: 'pgsql://oaipmh:secret@localhost:5432/oai_data_provider'
-# database: 'sqlite3:////home/oaipmh/database.db'
+# database: 'sqlite3:////opt/oaipmh/database.db'
 #
 # Run "composer doctrine:initialize-database" after switching to a new DB to
 # test the settings and initialize the database!
 #
 database: 'sqlite3:///%BASEDIR%/data/sqlite3.db'
 
+#
+# Memory Limit
+#
+# This defines the maximum amount of memory the indexing process should use per
+# run before it persists changes to the database, flushes it's internal storage
+# and starts over with the next batch of records. Higher settings result in
+# better performance, but depending on the size of your records even smaller
+# values may be needed to prevent running out of memory.
+# The value is interpreted as a percentage of the PHP "memory_limit".
+#
+# [0.2 - 0.8]
+#
+memoryLimit: 0.4
+
 #
 # Metadata formats, namespaces and schemas of your records
 #

src/Configuration.php

@@ -39,6 +39,7 @@ use Symfony\Component\Yaml\Yaml;
  * @property-read string $repositoryName
  * @property-read string $adminEmail
  * @property-read string $database
+ * @property-read float $memoryLimit
  * @property-read array $metadataPrefix
  * @property-read string $deletedRecords
  * @property-read int $maxRecords
@@ -86,6 +87,13 @@ class Configuration
                 new Assert\Type('string'),
                 new Assert\NotBlank()
             ],
+            'memoryLimit' => [
+                new Assert\Type('float'),
+                new Assert\Range([
+                    'min' => 0.2,
+                    'max' => 0.8
+                ])
+            ],
             'metadataPrefix' => [
                 new Assert\Type('array'),
                 new Assert\All([

src/Console.php

@@ -60,7 +60,7 @@ abstract class Console extends Command
      *
      * @return int The memory limit in bytes or -1 if unlimited
      */
-    protected function getMemoryLimit(): int
+    protected function getPhpMemoryLimit(): int
     {
         $ini = trim(ini_get('memory_limit'));
         $limit = (int) $ini;

src/Console/CsvImportCommand.php

@@ -23,6 +23,7 @@ declare(strict_types=1);
 namespace OCC\OaiPmh2\Console;
 
 use DateTime;
+use OCC\OaiPmh2\Configuration;
 use OCC\OaiPmh2\Console;
 use OCC\OaiPmh2\Database;
 use OCC\OaiPmh2\Entity\Format;
@@ -119,7 +120,8 @@ class CsvImportCommand extends Console
         if (!$this->validateInput($input, $output)) {
             return Command::INVALID;
         }
-        $memoryLimit = $this->getMemoryLimit();
+        $phpMemoryLimit = $this->getPhpMemoryLimit();
+        $memoryLimit = Configuration::getInstance()->memoryLimit;
 
         /** @var array<string, string> */
         $arguments = $input->getArguments();
@@ -161,8 +163,8 @@ class CsvImportCommand extends Console
             $progressIndicator->advance();
             $progressIndicator->setMessage('Importing... ' . (string) $count . ' records done.');
 
-            // Flush to database if memory usage reaches 30% of available limit.
+            // Flush to database if memory usage reaches limit.
-            if ((memory_get_usage() / $memoryLimit) > 0.3) {
+            if ((memory_get_usage() / $phpMemoryLimit) > $memoryLimit) {
                 Database::getInstance()->flush([Record::class]);
             }
         }