Browse Source

Merge pull request #569 from sebastian-meyer/fix-solarium-upgrade

Finish Solarium upgrade
pull/573/head
Sebastian Meyer 11 months ago
committed by GitHub
parent
commit
99a7502ac0
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      Classes/Common/DocumentList.php
  2. 4
      Classes/Common/Indexer.php
  3. 284
      Classes/Common/Solr.php
  4. 47
      Classes/Hooks/ConfigurationForm.php
  5. 114
      Classes/Hooks/DataHandler.php
  6. 4
      Classes/Plugin/Collection.php
  7. 50
      Classes/Plugin/Eid/SearchInDocument.php
  8. 34
      Classes/Plugin/Eid/SearchSuggest.php
  9. 4
      Classes/Plugin/OaiPmh.php
  10. 8
      Resources/Private/Language/FlashMessages.xml
  11. 8
      Resources/Private/Language/Labels.xml
  12. 94
      Resources/Public/Javascript/Search/SearchInDocument.js
  13. 10
      Resources/Public/Javascript/Search/Suggester.js
  14. 54
      class.ext_update.php
  15. 2
      composer.json
  16. 2
      ext_conf_template.txt

5
Classes/Common/DocumentList.php

@ -565,7 +565,10 @@ class DocumentList implements \ArrayAccess, \Countable, \Iterator, \TYPO3\CMS\Co
// Get Solr instance.
if (!$this->solr) {
// Connect to Solr server.
if ($this->solr = Solr::getInstance($this->metadata['options']['core'])) {
$solr = Solr::getInstance($this->metadata['options']['core']);
if ($solr->ready) {
$this->solr = $solr;
// Get indexed fields.
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('tx_dlf_metadata');

4
Classes/Common/Indexer.php

@ -560,7 +560,9 @@ class Indexer
// Get Solr instance.
if (!self::$solr) {
// Connect to Solr server.
if (self::$solr = Solr::getInstance($core)) {
$solr = Solr::getInstance($core);
if ($solr->ready) {
self::$solr = $solr;
// Load indexing configuration if needed.
if ($pid) {
self::loadIndexConf($pid);

284
Classes/Common/Solr.php

@ -12,8 +12,10 @@
namespace Kitodo\Dlf\Common;
use TYPO3\CMS\Core\Cache\CacheManager;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
/**
* Solr class for the 'dlf' extension
@ -23,22 +25,31 @@ use TYPO3\CMS\Core\Utility\GeneralUtility;
* @package TYPO3
* @subpackage dlf
* @access public
* @property-read string|null $core This holds the core name for the current instance
* @property-write int $cPid This holds the PID for the configuration
* @property int $limit This holds the max results
* @property-read int $numberOfHits This holds the number of hits for last search
* @property-write array $params This holds the additional query parameters
* @property-read bool $ready Is the search instantiated successfully?
* @property-read bool $ready Is the Solr service instantiated successfully?
* @property-read \Solarium\Client $service This holds the Solr service object
*/
class Solr
{
/**
* This holds the Solr configuration
*
* @var array
* @access protected
*/
protected $config = [];
/**
* This holds the core name
*
* @var string
* @var string|null
* @access protected
*/
protected $core = '';
protected $core = null;
/**
* This holds the PID for the configuration
@ -105,6 +116,54 @@ class Solr
protected $service;
/**
* Add a new core to Apache Solr
*
* @access public
*
* @param string $core: The name of the new core. If empty, the next available core name is used.
*
* @return string The name of the new core
*/
public static function createCore($core = '')
{
// Get next available core name if none given.
if (empty($core)) {
$core = 'dlfCore' . self::getNextCoreNumber();
}
// Get Solr service instance.
$solr = self::getInstance($core);
// Create new core if core with given name doesn't exist.
if ($solr->ready) {
// Core already exists.
return $core;
} else {
// Core doesn't exist yet.
$solrAdmin = self::getInstance();
if ($solrAdmin->ready) {
$query = $solrAdmin->service->createCoreAdmin();
$action = $query->createCreate();
$action->setConfigSet('dlf');
$action->setCore($core);
$action->setDataDir('data');
$action->setInstanceDir($core);
$query->setAction($action);
try {
$response = $solrAdmin->service->coreAdmin($query);
if ($response->getWasSuccessful()) {
// Core successfully created.
return $core;
}
} catch (\Exception $e) {
// Nothing to do here.
}
} else {
Helper::devLog('Apache Solr not available', DEVLOG_SEVERITY_ERROR);
}
}
return '';
}
/**
* Escape all special characters in a query string
*
* @access public
@ -115,13 +174,13 @@ class Solr
*/
public static function escapeQuery($query)
{
$helper = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\Solarium\Core\Query\Helper::class);
$helper = GeneralUtility::makeInstance(\Solarium\Core\Query\Helper::class);
// Escape query phrase or term.
if (preg_match('/^".*"$/', $query)) {
return $helper->escapePhrase(trim($query, '"'));
} else {
// Using a modified escape function here to retain whitespace, '*' and '?' for search truncation.
// @see https://github.com/solariumphp/solarium/blob/4.x/src/Core/Query/Helper.php#L68 for reference
// @see https://github.com/solariumphp/solarium/blob/5.x/src/Core/Query/Helper.php#L70 for reference
/* return $helper->escapeTerm($query); */
return preg_replace('/(\+|-|&&|\|\||!|\(|\)|\{|}|\[|]|\^|"|~|:|\/|\\\)/', '\\\$1', $query);
}
@ -175,11 +234,7 @@ class Solr
} else {
$query = self::escapeQuery($query);
}
} elseif (
!empty($query)
&& $query !== '*'
) {
// Don't escape plain asterisk search.
} else {
$query = self::escapeQuery($query);
}
return $query;
@ -190,112 +245,94 @@ class Solr
*
* @access public
*
* @param mixed $core: Name or UID of the core to load
* @param mixed $core: Name or UID of the core to load or null to get core admin endpoint
*
* @return \Kitodo\Dlf\Common\Solr Instance of this class
*/
public static function getInstance($core)
public static function getInstance($core = null)
{
// Get core name if UID is given.
if (\TYPO3\CMS\Core\Utility\MathUtility::canBeInterpretedAsInteger($core)) {
if (MathUtility::canBeInterpretedAsInteger($core)) {
$core = Helper::getIndexNameFromUid($core, 'tx_dlf_solrcores');
}
// Check if core is set.
if (empty($core)) {
Helper::devLog('Invalid core name "' . $core . '" for Apache Solr', DEVLOG_SEVERITY_ERROR);
return;
}
// Check if there is an instance in the registry already.
// Check if core is set or null.
if (
is_object(self::$registry[$core])
&& self::$registry[$core] instanceof self
empty($core)
&& $core !== null
) {
// Return singleton instance if available.
return self::$registry[$core];
Helper::devLog('Invalid core UID or name given for Apache Solr', DEVLOG_SEVERITY_ERROR);
}
if (!empty($core)) {
// Check if there is an instance in the registry already.
if (
is_object(self::$registry[$core])
&& self::$registry[$core] instanceof self
) {
// Return singleton instance if available.
return self::$registry[$core];
}
}
// Create new instance...
$instance = new self($core);
// ...and save it to registry.
if ($instance->ready) {
self::$registry[$core] = $instance;
// Return new instance.
return $instance;
} else {
Helper::devLog('Could not connect to Apache Solr server', DEVLOG_SEVERITY_ERROR);
return;
if (!empty($instance->core)) {
self::$registry[$instance->core] = $instance;
}
return $instance;
}
/**
* Returns the connection information for Solr
*
* @access public
*
* @return array The connection parameters for a specific Solr core
*/
public static function getSolrConnectionInfo()
{
$solrInfo = [];
// Extract extension configuration.
$conf = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][self::$extKey]);
// Derive Solr scheme
$solrInfo['scheme'] = empty($conf['solrHttps']) ? 'http' : 'https';
// Derive Solr host name.
$solrInfo['host'] = ($conf['solrHost'] ? $conf['solrHost'] : '127.0.0.1');
// Set username and password.
$solrInfo['username'] = $conf['solrUser'];
$solrInfo['password'] = $conf['solrPass'];
// Set port if not set.
$solrInfo['port'] = \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($conf['solrPort'], 1, 65535, 8983);
// Trim path of slashes.
$solrInfo['path'] = trim($conf['solrPath'], '/');
// Timeout
$solrInfo['timeout'] = \TYPO3\CMS\Core\Utility\MathUtility::forceIntegerInRange($conf['solrTimeout'], 1, intval(ini_get('max_execution_time')), 10);
return $solrInfo;
}
/**
* Returns the request URL for a specific Solr core
* Get next unused Solr core number
*
* @access public
*
* @param string $core: Name of the core to load
* @param int $number: Number to start with
*
* @return string The request URL for a specific Solr core
* @return int First unused core number found
*/
public static function getSolrUrl($core = '')
public static function getNextCoreNumber($number = 0)
{
// Get Solr connection information.
$solrInfo = self::getSolrConnectionInfo();
if (
$solrInfo['username']
&& $solrInfo['password']
) {
$host = $solrInfo['username'] . ':' . $solrInfo['password'] . '@' . $solrInfo['host'];
$number = max(intval($number), 0);
// Check if core already exists.
$solr = self::getInstance('dlfCore' . $number);
if (!$solr->ready) {
return $number;
} else {
$host = $solrInfo['host'];
return self::getNextCoreNumber($number + 1);
}
// Return entire request URL.
return $solrInfo['scheme'] . '://' . $host . ':' . $solrInfo['port'] . '/' . $solrInfo['path'] . '/solr/' . $core;
}
/**
* Get next unused Solr core number
*
* @access public
* Sets the connection information for Solr
*
* @param int $start: Number to start with
* @access protected
*
* @return int First unused core number found
* @return void
*/
public static function solrGetCoreNumber($start = 0)
protected function loadSolrConnectionInfo()
{
$start = max(intval($start), 0);
// Check if core already exists.
if (self::getInstance('dlfCore' . $start) === null) {
return $start;
} else {
return self::solrGetCoreNumber($start + 1);
if (empty($this->config)) {
$config = [];
// Extract extension configuration.
$conf = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf'][self::$extKey]);
// Derive Solr scheme
$config['scheme'] = empty($conf['solrHttps']) ? 'http' : 'https';
// Derive Solr host name.
$config['host'] = ($conf['solrHost'] ? $conf['solrHost'] : '127.0.0.1');
// Set username and password.
$config['username'] = $conf['solrUser'];
$config['password'] = $conf['solrPass'];
// Set port if not set.
$config['port'] = MathUtility::forceIntegerInRange($conf['solrPort'], 1, 65535, 8983);
// Trim path of slashes and (re-)add trailing slash if path not empty.
$config['path'] = trim($conf['solrPath'], '/');
if (!empty($config['path'])) {
$config['path'] .= '/';
}
// Set connection timeout lower than PHP's max_execution_time.
$max_execution_time = intval(ini_get('max_execution_time')) ?: 30;
$config['timeout'] = MathUtility::forceIntegerInRange($conf['solrTimeout'], 1, $max_execution_time, 10);
$this->config = $config;
}
}
@ -348,7 +385,7 @@ class Solr
];
}
// Save list of documents.
$list = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(DocumentList::class);
$list = GeneralUtility::makeInstance(DocumentList::class);
$list->reset();
$list->add(array_values($toplevel));
// Set metadata for search.
@ -392,7 +429,7 @@ class Solr
// calculate cache identifier
$cacheIdentifier = hash('md5', print_r(array_merge($this->params, $parameters), 1));
$cache = GeneralUtility::makeInstance(\TYPO3\CMS\Core\Cache\CacheManager::class)->getCache('tx_dlf_solr');
$cache = GeneralUtility::makeInstance(CacheManager::class)->getCache('tx_dlf_solr');
$resultSet = [];
if (($entry = $cache->get($cacheIdentifier)) === false) {
@ -411,6 +448,18 @@ class Solr
}
/**
* This returns $this->core via __get()
*
* @access protected
*
* @return string|null The core name of the current query endpoint or null if core admin endpoint
*/
protected function _getCore()
{
return $this->core;
}
/**
* This returns $this->limit via __get()
*
* @access protected
@ -565,37 +614,66 @@ class Solr
*
* @access protected
*
* @param string $core: The name of the core to use
* @param string|null $core: The name of the core to use or null for core admin endpoint
*
* @return void
*/
protected function __construct($core)
{
$solrInfo = self::getSolrConnectionInfo();
// Get Solr connection parameters from configuration.
$this->loadSolrConnectionInfo();
// Configure connection adapter.
$adapter = GeneralUtility::makeInstance(\Solarium\Core\Client\Adapter\Http::class);
$adapter->setTimeout($this->config['timeout']);
// Configure event dispatcher.
// Todo: When updating to TYPO3 >=10.x and Solarium >=6.x
// we have to provide an PSR-14 Event Dispatcher instead of
// "null".
// $eventDispatcher = GeneralUtility::makeInstance(\TYPO3\CMS\Core\EventDispatcher\EventDispatcher::class);
$eventDispatcher = null;
// Configure endpoint.
$config = [
'endpoint' => [
'dlf' => [
'scheme' => $solrInfo['scheme'],
'host' => $solrInfo['host'],
'port' => $solrInfo['port'],
'path' => '/' . $solrInfo['path'] . '/',
'default' => [
'scheme' => $this->config['scheme'],
'host' => $this->config['host'],
'port' => $this->config['port'],
'path' => '/' . $this->config['path'],
'core' => $core,
'username' => $solrInfo['username'],
'password' => $solrInfo['password'],
'timeout' => $solrInfo['timeout']
'username' => $this->config['username'],
'password' => $this->config['password']
]
]
];
// Instantiate Solarium\Client class.
$this->service = \TYPO3\CMS\Core\Utility\GeneralUtility::makeInstance(\Solarium\Client::class, $config);
$this->service = GeneralUtility::makeInstance(\Solarium\Client::class, $adapter, $eventDispatcher, $config);
// Check if connection is established.
$ping = $this->service->createPing();
$query = $this->service->createCoreAdmin();
$action = $query->createStatus();
if ($core !== null) {
$action->setCore($core);
}
$query->setAction($action);
try {
$this->service->ping($ping);
// Set core name.
$this->core = $core;
// Instantiation successful!
$this->ready = true;
$response = $this->service->coreAdmin($query);
if ($response->getWasSuccessful()) {
// Solr is reachable, but is the core as well?
if ($core !== null) {
$result = $response->getStatusResult();
if (
$result instanceof \Solarium\QueryType\Server\CoreAdmin\Result\StatusResult
&& $result->getUptime() > 0
) {
// Set core name.
$this->core = $core;
} else {
// Core not available.
return;
}
}
// Instantiation successful!
$this->ready = true;
}
} catch (\Exception $e) {
// Nothing to do here.
}

47
Classes/Hooks/ConfigurationForm.php

@ -46,43 +46,20 @@ class ConfigurationForm
*/
public function checkSolrConnection()
{
$solrInfo = Solr::getSolrConnectionInfo();
// Prepend username and password to hostname.
if (
!empty($solrInfo['username'])
&& !empty($solrInfo['password'])
) {
$host = $solrInfo['username'] . ':' . $solrInfo['password'] . '@' . $solrInfo['host'];
$solr = Solr::getInstance();
if ($solr->ready) {
Helper::addMessage(
htmlspecialchars($GLOBALS['LANG']->getLL('solr.status')),
htmlspecialchars($GLOBALS['LANG']->getLL('solr.connected')),
\TYPO3\CMS\Core\Messaging\FlashMessage::OK
);
} else {
$host = $solrInfo['host'];
}
// Build request URI.
$url = $solrInfo['scheme'] . '://' . $host . ':' . $solrInfo['port'] . '/' . $solrInfo['path'] . '/admin/cores?wt=xml';
$context = stream_context_create([
'http' => [
'method' => 'GET',
'user_agent' => (!empty($this->conf['useragent']) ? $this->conf['useragent'] : ini_get('user_agent'))
]
]);
// Try to connect to Solr server.
$response = @simplexml_load_string(file_get_contents($url, false, $context));
// Check status code.
if ($response) {
$status = $response->xpath('//lst[@name="responseHeader"]/int[@name="status"]');
if (is_array($status)) {
Helper::addMessage(
htmlspecialchars(sprintf($GLOBALS['LANG']->getLL('solr.status'), (string) $status[0])),
htmlspecialchars($GLOBALS['LANG']->getLL('solr.connected')),
($status[0] == 0 ? \TYPO3\CMS\Core\Messaging\FlashMessage::OK : \TYPO3\CMS\Core\Messaging\FlashMessage::WARNING)
);
return Helper::renderFlashMessages();
}
Helper::addMessage(
htmlspecialchars($GLOBALS['LANG']->getLL('solr.error')),
htmlspecialchars($GLOBALS['LANG']->getLL('solr.notConnected')),
\TYPO3\CMS\Core\Messaging\FlashMessage::WARNING
);
}
Helper::addMessage(
htmlspecialchars(sprintf($GLOBALS['LANG']->getLL('solr.error'), $url)),
htmlspecialchars($GLOBALS['LANG']->getLL('solr.notConnected')),
\TYPO3\CMS\Core\Messaging\FlashMessage::WARNING
);
return Helper::renderFlashMessages();
}

114
Classes/Hooks/DataHandler.php

@ -91,53 +91,13 @@ class DataHandler
break;
// Field post-processing for table "tx_dlf_solrcores".
case 'tx_dlf_solrcores':
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('tx_dlf_solrcores');
// Get number of existing cores.
$result = $queryBuilder
->select('*')
->from('tx_dlf_solrcores')
->execute();
// Get first unused core number.
$coreNumber = Solr::solrGetCoreNumber(count($result->fetchAll()));
// Get Solr credentials.
$conf = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['dlf']);
$solrInfo = Solr::getSolrConnectionInfo();
// Prepend username and password to hostname.
if (
$solrInfo['username']
&& $solrInfo['password']
) {
$host = $solrInfo['username'] . ':' . $solrInfo['password'] . '@' . $solrInfo['host'];
} else {
$host = $solrInfo['host'];
// Create new Solr core.
$fieldArray['index_name'] = Solr::createCore();
if (empty($fieldArray['index_name'])) {
Helper::devLog('Could not create new Apache Solr core', DEVLOG_SEVERITY_ERROR);
// Solr core could not be created, thus unset field array.
$fieldArray = [];
}
$context = stream_context_create([
'http' => [
'method' => 'GET',
'user_agent' => ($conf['useragent'] ? $conf['useragent'] : ini_get('user_agent'))
]
]);
// Build request for adding new Solr core.
// @see http://wiki.apache.org/solr/CoreAdmin
$url = $solrInfo['scheme'] . '://' . $host . ':' . $solrInfo['port'] . '/' . $solrInfo['path'] . '/admin/cores?wt=xml&action=CREATE&name=dlfCore' . $coreNumber . '&instanceDir=dlfCore' . $coreNumber . '&dataDir=data&configSet=dlf';
$response = @simplexml_load_string(file_get_contents($url, false, $context));
// Process response.
if ($response) {
$solrStatus = $response->xpath('//lst[@name="responseHeader"]/int[@name="status"]');
if (
is_array($solrStatus)
&& $solrStatus[0] == 0
) {
$fieldArray['index_name'] = 'dlfCore' . $coreNumber;
return;
}
}
Helper::devLog('Could not create new Apache Solr core "dlfCore' . $coreNumber . '"', DEVLOG_SEVERITY_ERROR);
// Solr core could not be created, thus unset field array.
$fieldArray = [];
break;
}
} elseif ($status == 'update') {
@ -286,7 +246,8 @@ class DataHandler
$resArray = $allResults[0];
if ($resArray['hidden']) {
// Establish Solr connection.
if ($solr = Solr::getInstance($resArray['core'])) {
$solr = Solr::getInstance($resArray['core']);
if ($solr->ready) {
// Delete Solr document.
$updateQuery = $solr->service->createUpdate();
$updateQuery->addDeleteQuery('uid:' . $id);
@ -326,10 +287,10 @@ class DataHandler
in_array($command, ['move', 'delete', 'undelete'])
&& $table == 'tx_dlf_documents'
) {
// Get Solr-Core.
// Get Solr core.
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('tx_dlf_solrcores');
// tx_dlf_documents is already deleted at this point --> include deleted documents in query
// Record in "tx_dlf_documents" is already deleted at this point.
$queryBuilder
->getRestrictions()
->removeByType(DeletedRestriction::class);
@ -365,7 +326,8 @@ class DataHandler
case 'move':
case 'delete':
// Establish Solr connection.
if ($solr = Solr::getInstance($resArray['core'])) {
$solr = Solr::getInstance($resArray['core']);
if ($solr->ready) {
// Delete Solr document.
$updateQuery = $solr->service->createUpdate();
$updateQuery->addDeleteQuery('uid:' . $id);
@ -387,5 +349,57 @@ class DataHandler
}
}
}
if (
$command === 'delete'
&& $table == 'tx_dlf_solrcores'
) {
// Is core deletion allowed in extension configuration?
$extConf = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['dlf']);
if (!empty($extConf['solrAllowCoreDelete'])) {
// Delete core from Apache Solr as well.
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)
->getQueryBuilderForTable('tx_dlf_solrcores');
// Record in "tx_dlf_solrcores" is already deleted at this point.
$queryBuilder
->getRestrictions()
->removeByType(DeletedRestriction::class);
$result = $queryBuilder
->select(
'tx_dlf_solrcores.index_name AS core'
)
->from('tx_dlf_solrcores')
->where($queryBuilder->expr()->eq('tx_dlf_solrcores.uid', intval($id)))
->setMaxResults(1)
->execute();
$allResults = $result->fetchAll();
if (count($allResults) == 1) {
$resArray = $allResults[0];
// Establish Solr connection.
$solr = Solr::getInstance();
if ($solr->ready) {
// Delete Solr core.
$query = $solr->service->createCoreAdmin();
$action = $query->createUnload();
$action->setCore($resArray['core']);
$action->setDeleteDataDir(true);
$action->setDeleteIndex(true);
$action->setDeleteInstanceDir(true);
$query->setAction($action);
try {
$response = $solr->service->coreAdmin($query);
if ($response->getWasSuccessful()) {
return;
}
} catch (\Exception $e) {
// Nothing to do here.
}
}
Helper::devLog('Core ' . $resArray['core'] . ' could not be deleted from Apache Solr', DEVLOG_SEVERITY_WARNING);
}
}
}
}
}

4
Classes/Plugin/Collection.php

@ -142,6 +142,10 @@ class Collection extends \Kitodo\Dlf\Common\AbstractPlugin
$this->showSingleCollection(intval($resArray['uid']));
}
$solr = Solr::getInstance($this->conf['solrcore']);
if (!$solr->ready) {
Helper::devLog('Apache Solr not available', DEVLOG_SEVERITY_ERROR);
return $content;
}
// We only care about the UID and partOf in the results and want them sorted
$params['fields'] = 'uid,partof';
$params['sort'] = ['uid' => 'asc'];

50
Classes/Plugin/Eid/SearchInDocument.php

@ -14,7 +14,6 @@ namespace Kitodo\Dlf\Plugin\Eid;
use Kitodo\Dlf\Common\Helper;
use Kitodo\Dlf\Common\Solr;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use TYPO3\CMS\Core\Http\Response;
@ -38,29 +37,48 @@ class SearchInDocument
*/
public function main(ServerRequestInterface $request)
{
$output = [
'documents' => [],
'numFound' => 0
];
// Get input parameters and decrypt core name.
$parameters = $request->getParsedBody();
$encrypted = (string)$parameters['encrypted'];
$hashed = (string)$parameters['hashed'];
$encrypted = (string) $parameters['encrypted'];
$hashed = (string) $parameters['hashed'];
$count = intval($parameters['start']);
if (empty($encrypted) || empty($hashed)) {
throw new \InvalidArgumentException('No valid parameter passed!', 1580585079);
}
$core = Helper::decrypt($encrypted, $hashed);
$output = '';
if (!empty($core)) {
$query = (string)$parameters['q'];
$uid = (string)$parameters['uid'];
$start = (string)$parameters['start'];
$url = trim(Solr::getSolrUrl($core), '/') . '/select?wt=json&q=fulltext:(' . Solr::escapeQuery($query) . ')%20AND%20uid:' . $uid
. '&hl=on&hl.fl=fulltext&fl=uid,id,page&hl.method=fastVector'
. '&start=' . $start . '&rows=20';
$output = GeneralUtility::getUrl($url);
// Perform Solr query.
$solr = Solr::getInstance($core);
if ($solr->ready) {
$query = $solr->service->createSelect();
$query->setFields(['id', 'uid', 'page']);
$query->setQuery('fulltext:(' . Solr::escapeQuery((string) $parameters['q']) . ') AND uid:' . intval($parameters['uid']));
$query->setStart($count)->setRows(20);
$hl = $query->getHighlighting();
$hl->setFields(['fulltext']);
$hl->setUseFastVectorHighlighter(true);
$results = $solr->service->select($query);
$output['numFound'] = $results->getNumFound();
$highlighting = $results->getHighlighting();
foreach ($results as $result) {
$snippet = $highlighting->getResult($result->id)->getField('fulltext');
$document = [
'id' => $result->id,
'uid' => $result->uid,
'page' => $result->page,
'snippet' => !empty($snippet) ? implode(' [...] ', $snippet) : ''
];
$output['documents'][$count] = $document;
$count++;
}
}
// create response object
// Create response object.
/** @var Response $response */
$response = GeneralUtility::makeInstance(Response::class);
$response->getBody()->write($output);
$response->getBody()->write(json_encode($output));
return $response;
}
}

34
Classes/Plugin/Eid/SearchSuggest.php

@ -14,7 +14,6 @@ namespace Kitodo\Dlf\Plugin\Eid;
use Kitodo\Dlf\Common\Helper;
use Kitodo\Dlf\Common\Solr;
use Psr\Http\Message\ResponseInterface;
use Psr\Http\Message\ServerRequestInterface;
use TYPO3\CMS\Core\Http\Response;
@ -39,25 +38,36 @@ class SearchSuggest
*/
public function main(ServerRequestInterface $request)
{
$output = [];
// Get input parameters and decrypt core name.
$parameters = $request->getParsedBody();
$encrypted = (string)$parameters['encrypted'];
$hashed = (string)$parameters['hashed'];
$encrypted = (string) $parameters['encrypted'];
$hashed = (string) $parameters['hashed'];
if (empty($encrypted) || empty($hashed)) {
throw new \InvalidArgumentException('No valid parameter passed!', 1580585079);
}
$core = Helper::decrypt($encrypted, $hashed);
$output = '';
if (!empty($core)) {
$query = (string)$parameters['q'];
$url = trim(Solr::getSolrUrl($core), '/') . '/suggest/?wt=xml&q=' . Solr::escapeQuery($query);
$output = GeneralUtility::getUrl($url);
// Perform Solr query.
$solr = Solr::getInstance($core);
if ($solr->ready) {
$query = $solr->service->createSelect();
$query->setHandler('suggest');
$query->setQuery(Solr::escapeQuery((string) $parameters['q']));
$query->setRows(0);
$results = $solr->service->select($query)->getResponse()->getBody();
$result = json_decode($results);
foreach ($result->spellcheck->suggestions as $suggestions) {
if (is_object($suggestions)) {
foreach ($suggestions->suggestion as $suggestion) {
$output[] = $suggestion;
}
}
}
}
// create response object
// Create response object.
/** @var Response $response */
$response = GeneralUtility::makeInstance(Response::class);
$response->getBody()->write($output);
$response->getBody()->write(json_encode($output));
return $response;
}
}

4
Classes/Plugin/OaiPmh.php

@ -923,6 +923,10 @@ class OaiPmh extends \Kitodo\Dlf\Common\AbstractPlugin
$solr_query .= ' AND timestamp:[' . $from . ' TO ' . $until . ']';
$documentSet = [];
$solr = Solr::getInstance($this->conf['solrcore']);
if (!$solr->ready) {
Helper::devLog('Apache Solr not available', DEVLOG_SEVERITY_ERROR);
return $documentSet;
}
if (intval($this->conf['solr_limit']) > 0) {
$solr->limit = intval($this->conf['solr_limit']);
}

8
Resources/Private/Language/FlashMessages.xml

@ -76,9 +76,9 @@
<label index="update.documentSetFormatForOldEntriesOkay">...successfully completed!</label>
<label index="update.documentSetFormatForOldEntriesNotOkay">...incomplete! Please set missing tx_dlf_document.document_format values to 'METS'.</label>
<label index="solr.connected">Connection established!</label>
<label index="solr.status">The status code returned by Apache Solr is &lt;strong&gt;%s&lt;/strong&gt;.</label>
<label index="solr.status">Apache Solr is running and taking requests.</label>
<label index="solr.notConnected">Connection failed!</label>
<label index="solr.error">Apache Solr was not reachable under &lt;strong&gt;%s&lt;/strong&gt;.</label>
<label index="solr.error">Apache Solr not reachable with given configuration.</label>
<label index="metadataFormats.nsOkay">Default namespaces found!</label>
<label index="metadataFormats.nsOkayMsg">The namespace definitions for MODS, TEIHDR and ALTO do exist.</label>
<label index="metadataFormats.nsCreated">Default namespaces created successfully!</label>
@ -148,9 +148,9 @@
<label index="update.documentSetFormatForOldEntriesOkay">...erfolgreich beendet!</label>
<label index="update.documentSetFormatForOldEntriesNotOkay">...abgebrochen! Bitte in der Tabelle tx_dlf_document.document_format 'METS'.</label>
<label index="solr.connected">Verbindung hergestellt!</label>
<label index="solr.status">Apache Solr gibt den Statuscode &lt;strong&gt;%s&lt;/strong&gt; zurück.</label>
<label index="solr.status">Apache Solr ist verfügbar und nimmt Anfragen entgegen.</label>
<label index="solr.notConnected">Verbindung fehlgeschlagen!</label>
<label index="solr.error">Apache Solr ist unter &lt;strong&gt;%s&lt;/strong&gt; nicht erreichbar.</label>
<label index="solr.error">Apache Solr ist mit der gegebenen Konfiguration nicht erreichbar.</label>
<label index="metadataFormats.nsOkay">Standard-Namensräume gefunden!</label>
<label index="metadataFormats.nsOkayMsg">Die Standard-Namensräume für MODS, TEIHDR und ALTO sind definiert.</label>
<label index="metadataFormats.nsCreated">Standard-Namensräume erfolgreich angelegt!</label>

8
Resources/Private/Language/Labels.xml

@ -190,13 +190,14 @@
<label index="config.iiifThumbnailWidth">Maximum thumbnail width for IIIF images: Only for images without a thumbnail declaration (default is "150")</label>
<label index="config.iiifThumbnailHeight">Maximum thumbnail height for IIIF images: Only for images without a thumbnail declaration (default is "150")</label>
<label index="config.solrConnect">Solr Connection</label>
<label index="config.solrHttps">Use https: (default is "FALSE")</label>
<label index="config.solrHttps">Use HTTPS: (default is "FALSE")</label>
<label index="config.solrHost">Solr Server Host: (default is "localhost")</label>
<label index="config.solrPort">Solr Server Port: (default is "8983")</label>
<label index="config.solrPath">Solr Server Path: without API endpoint "/solr" (default is "/")</label>
<label index="config.solrUser">Solr Server User: (default is "")</label>
<label index="config.solrPass">Solr Server Password: (default is "")</label>
<label index="config.solrTimeout">Solr Server Timeout: (default is "10")</label>
<label index="config.solrAllowCoreDelete">Allow Solr Core Deletion?: If a Solr Core is deleted in the TYPO3 Backend, should it be deleted in Apache Solr as well? (Standard ist "FALSE")</label>
</languageKey>
<languageKey index="de" type="array">
<label index="tx_dlf_actionlog">Aktionsprotokoll</label>
@ -292,7 +293,7 @@
<label index="tx_dlf_solrcores">Solr Kerne</label>
<label index="tx_dlf_solrcores.label">Anzeigeform</label>
<label index="tx_dlf_solrcores.index_name">Solr-Bezeichnung</label>
<label index="tx_dlf_solrcores.tab1">General</label>
<label index="tx_dlf_solrcores.tab1">Allgemein</label>
<label index="tx_dlf_collections">Sammlungen</label>
<label index="tx_dlf_collections.label">Anzeigeform</label>
<label index="tx_dlf_collections.index_name">Index-Bezeichnung</label>
@ -374,13 +375,14 @@
<label index="config.iiifThumbnailWidth">Maximale Thumbnail-Breite für IIIF-Images: Gilt nur für Bilder ohne Thumbnail-Angaben (Standard ist "150")</label>
<label index="config.iiifThumbnailHeight">Maximale Thumbnail-Höhe für IIIF-Images: Gilt nur für Bilder ohne Thumbnail-Angaben (Standard ist "150")</label>
<label index="config.solrConnect">Solr Verbindung</label>
<label index="config.solrHttps">Https verwenden: (Standard ist "FALSE")</label>
<label index="config.solrHttps">HTTPS verwenden: (Standard ist "FALSE")</label>
<label index="config.solrHost">Solr Server Host: (Standard ist "localhost")</label>
<label index="config.solrPort">Solr Server Port: (Standard ist "8983")</label>
<label index="config.solrPath">Solr Server Pfad: ohne API-Endpunkt "/solr" (Standard ist "/")</label>
<label index="config.solrUser">Solr Server Benutzername: (Standard ist "")</label>
<label index="config.solrPass">Solr Server Kennwort: (Standard ist "")</label>
<label index="config.solrTimeout">Solr Server Timeout: (Standard ist "10")</label>
<label index="config.solrAllowCoreDelete">Löschen von Solr Kern zulassen?: Soll beim Löschen eines Solr Kerns im TYPO3 Backend auch der entsprechende Index in Apache Solr gelöscht werden? (Standard ist "FALSE")</label>
</languageKey>
</data>
</T3locallang>

94
Resources/Public/Javascript/Search/SearchInDocument.js

@ -12,12 +12,11 @@
* This function increases the start parameter of the search form and submits
* the form.
*
* @param rows
* @returns void
*/
function nextResultPage(rows) {
function nextResultPage() {
var currentstart = $("#tx-dlf-search-in-document-form input[name='tx_dlf[start]']").val();
var newstart = parseInt(currentstart) + rows;
var newstart = parseInt(currentstart) + 20;
$("#tx-dlf-search-in-document-form input[name='tx_dlf[start]']").val(newstart);
$('#tx-dlf-search-in-document-form').submit();
};
@ -26,13 +25,11 @@ function nextResultPage(rows) {
* This function decreases the start parameter of the search form and submits
* the form.
*
* @param rows
* @returns void
*/
function previousResultPage(rows) {
function previousResultPage() {
var currentstart = $("#tx-dlf-search-in-document-form input[name='tx_dlf[start]']").val();
currentstart = parseInt(currentstart);
var newstart = (currentstart > rows) ? (currentstart - rows) : 0;
var newstart = (parseInt(currentstart) > 20) ? (parseInt(currentstart) - 20) : 0;
$("#tx-dlf-search-in-document-form input[name='tx_dlf[start]']").val(newstart);
$('#tx-dlf-search-in-document-form').submit();
};
@ -67,55 +64,60 @@ $(document).ready(function() {
hashed: $( "input[name='tx_dlf[hashed]']" ).val(),
},
function(data) {
var resultItems = [];
var resultList = '<div class="results-active-indicator"></div><ul>';
if (data.error) {
resultList += '<li class="error">' + data.error + '</li>';
} else {
for (var i=0; i < data.response.docs.length; i++) {
var resultItems = [];
var resultList = '<div class="results-active-indicator"></div><ul>';
var start = -1;
if (data['numFound'] > 0) {
// Take the workview baseUrl from the form action.
// The URL may be in the following form
// - http://example.com/index.php?id=14
// - http://example.com/workview (using slug on page with uid=14)
var baseUrl = $("form#tx-dlf-search-in-document-form").attr('action');
// Take the workview baseUrl from the form action.
// The URL may be in the following form
// - http://example.com/index.php?id=14
// - http://example.com/workview (using slug on page with uid=14)
var baseUrl = $("form#tx-dlf-search-in-document-form").attr('action');
if (baseUrl.indexOf('?')>0) {
if (baseUrl.indexOf('?') > 0) {
baseUrl += '&';
} else {
} else {
baseUrl += '?';
}
var searchHit = data.highlighting[data.response.docs[i].id].fulltext.toString();
searchHit = searchHit.substring(searchHit.indexOf('<em>')+4,searchHit.indexOf('</em>'));
}
data['documents'].forEach(function (element, i) {
if (start < 0) {
start = i;
}
var searchWord = element['snippet'];
searchWord = searchWord.substring(searchWord.indexOf('<em>') + 4, searchWord.indexOf('</em>'));
var newlink = baseUrl
+ 'tx_dlf[id]=' + data.response.docs[i].uid
+ '&tx_dlf[highlight_word]=' + encodeURIComponent(searchHit)
+ '&tx_dlf[page]=' + data.response.docs[i].page;
var link = baseUrl
+ 'tx_dlf[id]=' + element['uid']
+ '&tx_dlf[highlight_word]=' + encodeURIComponent(searchWord)
+ '&tx_dlf[page]=' + element['page'];
if (data.highlighting[data.response.docs[i].id].fulltext) {
resultItems[data.response.docs[i].page] = '<span class="structure">' + $('#tx-dlf-search-in-document-label-page').text() + ' ' + data.response.docs[i].page + '</span><br /><span ="textsnippet"><a href=\"' + newlink + '\">' + data.highlighting[data.response.docs[i].id].fulltext + '</a></span>';
}
}
if (resultItems.length > 0) {
// sort by page as this cannot be done with current solr schema
resultItems.sort(function(a, b){return a-b});
resultItems.forEach(function(item, index){
if (element['snippet'].length > 0) {
resultItems[element['page']] = '<span class="structure">'
+ $('#tx-dlf-search-in-document-label-page').text() + ' ' + element['page']
+ '</span><br />'
+ '<span ="textsnippet">'
+ '<a href=\"' + link + '\">' + element['snippet'] + '</a>'
+ '</span>';
}
});
// Sort result by page.
resultItems.sort(function (a, b) {
return a - b;
});
resultItems.forEach(function (item, index) {
resultList += '<li>' + item + '</li>';
});
} else {
resultList += '<li class="noresult">' + $('#tx-dlf-search-in-document-label-noresult').text() + '</li>';
}
}
resultList += '</ul>';
if (parseInt(data.response.start) > 0) {
resultList += '<input type="button" id="tx-dlf-search-in-document-button-previous" class="button-previous" onclick="previousResultPage(' + data.responseHeader.params.rows + ');" value="' + $('#tx-dlf-search-in-document-label-previous').text() + '" />';
}
if (parseInt(data.response.numFound) > (parseInt(data.response.start) + parseInt(data.responseHeader.params.rows))) {
resultList += '<input type="button" id="tx-dlf-search-in-document-button-next" class="button-next" onclick="nextResultPage(' + data.responseHeader.params.rows + ');" value="' + $('#tx-dlf-search-in-document-label-next').text() + '" />';
}
$('#tx-dlf-search-in-document-results').html(resultList);
resultList += '</ul>';
if (start > 0) {
resultList += '<input type="button" id="tx-dlf-search-in-document-button-previous" class="button-previous" onclick="previousResultPage();" value="' + $('#tx-dlf-search-in-document-label-previous').text() + '" />';
}
if (data['numFound'] > (start + 20)) {
resultList += '<input type="button" id="tx-dlf-search-in-document-button-next" class="button-next" onclick="nextResultPage();" value="' + $('#tx-dlf-search-in-document-label-next').text() + '" />';
}
$('#tx-dlf-search-in-document-results').html(resultList);
},
"json"
)

10
Resources/Public/Javascript/Search/Suggester.js

@ -30,15 +30,13 @@ $(
},
function(data) {
var result = [];
var option = "";
$("arr[name='suggestion'] str", data).each(function(i) {
option = $(this).text();
option = option.replace(/(\?|!|:|\\)/g, "\\\$1");
result.push(option);
data.forEach(function(element, index) {
element = element.replace(/(\?|!|:|\\)/g, "\\\$1");
result.push(element);
});
return response(result);
},
"xml");
"json");
},
minLength: 3,
appendTo: "#tx-dlf-search-suggest"

54
class.ext_update.php

@ -395,64 +395,34 @@ class ext_update
*/
protected function doSolariumSolrUpdate(): void
{
$error = false;
// Get all Solr cores that were not deleted.
$queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('tx_dlf_solrcores');
$result = $queryBuilder
->select('index_name')
->from('tx_dlf_solrcores')
->where('1=1')
->execute();
while ($resArray = $result->fetch()) {
// Instantiate search object.
$solr = Solr::getInstance($resArray['index_name']);
if (!$solr->ready) {
$conf = unserialize($GLOBALS['TYPO3_CONF_VARS']['EXT']['extConf']['dlf']);
$solrInfo = Solr::getSolrConnectionInfo();
// Prepend username and password to hostname.
if (
$solrInfo['username']
&& $solrInfo['password']
) {
$host = $solrInfo['username'] . ':' . $solrInfo['password'] . '@' . $solrInfo['host'];
} else {
$host = $solrInfo['host'];
}
$context = stream_context_create([
'http' => [
'method' => 'GET',
'user_agent' => ($conf['useragent'] ? $conf['useragent'] : ini_get('user_agent'))
]
]);
// Build request for adding new Solr core.
// @see http://wiki.apache.org/solr/CoreAdmin
$url = $solrInfo['scheme'] . '://' . $host . ':' . $solrInfo['port'] . '/' . $solrInfo['path'] . '/admin/cores?wt=xml&action=CREATE&name=' . $resArray['index_name'] . '&instanceDir=' . $resArray['index_name'] . '&dataDir=data&configSet=dlf';
$response = @simplexml_load_string(file_get_contents($url, false, $context));
// Process response.
if ($response) {
$status = $response->xpath('//lst[@name="responseHeader"]/int[@name="status"]');
if (
$status !== false
&& $status[0] === 0
) {
continue;
}
}
// Create core if it doesn't exist.
if (Solr::createCore($resArray['index_name']) !== $resArray['index_name']) {
Helper::addMessage(
htmlspecialchars($GLOBALS['LANG']->getLL('update.solariumSolrUpdateNotOkay')),
htmlspecialchars(sprintf($GLOBALS['LANG']->getLL('update.solariumSolrUpdate'), $resArray['index_name'])),
\TYPO3\CMS\Core\Messaging\FlashMessage::ERROR
);
$this->content .= Helper::renderFlashMessages();
return;
$error = true;
}
}
Helper::addMessage(
htmlspecialchars($GLOBALS['LANG']->getLL('update.solariumSolrUpdateOkay')),
htmlspecialchars($GLOBALS['LANG']->getLL('update.solariumSolrUpdate')),
\TYPO3\CMS\Core\Messaging\FlashMessage::OK
);
$this->content .= Helper::renderFlashMessages();
if (!$error) {
Helper::addMessage(
htmlspecialchars($GLOBALS['LANG']->getLL('update.solariumSolrUpdateOkay')),
htmlspecialchars($GLOBALS['LANG']->getLL('update.solariumSolrUpdate')),
\TYPO3\CMS\Core\Messaging\FlashMessage::OK
);
$this->content .= Helper::renderFlashMessages();
}
}
/**

2
composer.json

@ -35,7 +35,7 @@
"typo3/cms-core": "~8.7.32|~9.5.17",
"typo3/cms-tstemplate": "~8.7.32|~9.5.17",
"ubl/php-iiif-prezi-reader": "0.3.0",
"solarium/solarium": "^5.1.6"
"solarium/solarium": "^5.2"
},
"replace": {
"typo3-ter/dlf": "self.version"

2
ext_conf_template.txt

@ -44,3 +44,5 @@ solrUser =
solrPass =
# cat=Solr; type=string; label=LLL:EXT:dlf/Resources/Private/Language/Labels.xml:config.solrTimeout
solrTimeout = 10
# cat=Solr; type=boolean; label=LLL:EXT:dlf/Resources/Private/Language/Labels.xml:config.solrAllowCoreDelete
solrAllowCoreDelete = 0

Loading…
Cancel
Save