Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Relax Version Dependencies for recent Neos and Elasticsearch Versions #1

Draft
wants to merge 22 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
42c0819
Relax Version Dependencies for recent Neos and Elasticsearch Versions
mficzel Jun 30, 2021
8ac1c4b
WIP: Adjust interfaces
mficzel Jun 30, 2021
c300fee
WIP Work on search updates
mficzel Jun 30, 2021
433a75d
Adjust mapping to ES6+
Jun 30, 2021
cf93a05
Merge branch 'neos-5-and-7' of https://github.com/Flowpack/elasticsea…
Jun 30, 2021
c4c2000
Adjust to new API
Jun 30, 2021
f1d4124
WIP: Adjust query builder and query for new elastic mapping, added se…
mficzel Jul 1, 2021
243b4a8
WIP: Properly distribute nodes among indices
Jul 1, 2021
9f70797
WIP: Adjust output for better readability
mficzel Jul 1, 2021
4c91335
TASK: Improve feedback and respect limit argument
mficzel Jul 2, 2021
56d7fa1
TASK: Use batch size from settings
mficzel Jul 2, 2021
ecbb338
TASK: Use legacy dimension value combinations from cr adpter package …
mficzel Jul 2, 2021
f26aab1
TASK: Rename method
mficzel Jul 2, 2021
68d9bfa
Properly index dimension fallbacks
Jul 2, 2021
cabe47f
TASK: Index fulltext only if needed an flush bulk requests in Indexer
mficzel Jul 2, 2021
bbb70e8
Introduce workspace indexing modes for performance tuning
Jul 2, 2021
4c51655
Merge branch 'neos-5-and-7' of https://github.com/Flowpack/elasticsea…
Jul 2, 2021
d796e33
fix diagram
Jul 2, 2021
be4ae2d
TASK: Remove legacy commandcontroller and settings that are configure…
mficzel Jul 5, 2021
e7b7479
Properly reset the current bulk request
Jul 5, 2021
95facb5
Declare hierarchy relations as nested
Jul 13, 2021
5ae84dd
Properly compare DSPs when determining hierarchy relations
Jul 13, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
561 changes: 477 additions & 84 deletions Classes/Command/GraphIndexCommandController.php

Large diffs are not rendered by default.

102 changes: 102 additions & 0 deletions Classes/Domain/WorkspaceIndexingMode.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
<?php
namespace Flowpack\ElasticSearch\ContentGraphAdapter\Domain;

/*
* This file is part of the Flowpack.ElasticSearch.ContentGraphAdapter package.
*
* (c) Contributors of the Neos Project - www.neos.io
*
* This package is Open Source Software. For the full copyright and license
* information, please view the LICENSE file which was distributed with this
* source code.
*/

use Neos\ContentRepository\InMemoryGraph\NodeAggregate\Node;
use Neos\Flow\Annotations as Flow;

/**
* The workspace indexing mode
* @Flow\Proxy(false)
*/
final class WorkspaceIndexingMode
{
/**
* In this mode, only nodes in workspace live are indexed.
* Really fast but does not work on unpublished changes.
*
* ┌────────┐
* │ P:live │
* └────────┘
* ╱ ╲
* ╱ ╲
* ┌────▼────┐ ┌─────────────┐
* │ C:live │ │ C:user-me │ <--- not indexed
* └─────────┘ └─────────────┘
*/
const MODE_ONLY_LIVE = 'onlyLive';

/**
* In this mode, nodes are only indexed in their origin workspace.
* Still quite fast but does not work on unpublished changes in edge cases.
*
* ┌────────────────┐
* │ P:live,user-me │ <--- visible in user-me only due to fallback
* └────────────────┘ thus only indexed in live, visible in user-me
* ╱ ╲ but with upserted content not from C:user-me but from C:live
* ╱ ╲
* ┌────▼────┐ ┌───────▼─────┐
* indexed in live ---> │ C:live │ │ C:user-me │ <--- indexed in user-me
* └─────────┘ └─────────────┘
*/
const MODE_ONLY_ORIGIN = 'onlyOrigin';

/**
* In this mode, nodes are indexed in all their covered workspaces.
* Not so fast due to creation of large amounts of duplicates.
*
* ┌────────────────┐
* │ P:live,user-me │ <--- indexed in both live and user-me
* └────────────────┘
* ╱ ╲
* ╱ ╲
* ┌────▼────┐ ┌───────▼─────┐
* indexed in live ---> │ C:live │ │ C:user-me │ <--- indexed in user-me
* └─────────┘ └─────────────┘
*/
const MODE_FULL = 'full';

/**
* @var string
*/
private $value;

private function __construct(string $value)
{
$this->value = $value;
}

public static function fromString(string $value): self
{
if ($value !== self::MODE_ONLY_LIVE && $value !== self::MODE_ONLY_ORIGIN && $value !== self::MODE_FULL) {
throw WorkspaceIndexingModeIsInvalid::becauseItIsNoneOfTheDefinedValues($value);
}

return new self($value);
}

public function isNodeToBeIndexed(Node $node): bool
{
switch ($this->value) {
case self::MODE_ONLY_LIVE:
return $node->getWorkspace()->getName() === 'live';
default:
return true;
}
}

public function getValue(): string
{
return $this->value;
}
}

27 changes: 27 additions & 0 deletions Classes/Domain/WorkspaceIndexingModeIsInvalid.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<?php
namespace Flowpack\ElasticSearch\ContentGraphAdapter\Domain;

/*
* This file is part of the Flowpack.ElasticSearch.ContentGraphAdapter package.
*
* (c) Contributors of the Neos Project - www.neos.io
*
* This package is Open Source Software. For the full copyright and license
* information, please view the LICENSE file which was distributed with this
* source code.
*/

use Neos\Flow\Annotations as Flow;

/**
* The exception to be thrown if an invalid workspace indexing mode was attempted to be initialized
* @Flow\Proxy(false)
*/
final class WorkspaceIndexingModeIsInvalid extends \DomainException
{
public static function becauseItIsNoneOfTheDefinedValues(string $attemptedValue): self
{
return new self('Given value "' . $attemptedValue . '" is no valid workspace indexing mode, must be one of the defined constants.', 1625245635);
}
}

147 changes: 147 additions & 0 deletions Classes/Eel/ElasticSearchQuery.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
<?php
namespace Flowpack\ElasticSearch\ContentGraphAdapter\Eel;

/*
* This file is part of the Flowpack.ElasticSearch.ContentGraphAdapter package.
*
* (c) Contributors of the Neos Project - www.neos.io
*
* This package is Open Source Software. For the full copyright and license
* information, please view the LICENSE file which was distributed with this
* source code.
*/

use Neos\Flow\Annotations as Flow;
use Neos\Flow\Security\Context as SecurityContext;
use Neos\ContentRepository\Domain\Model\NodeInterface;
use Neos\ContentRepository\Domain\Service\ContentDimensionPresetSourceInterface;
use Neos\ContentRepository\Search\Search\QueryBuilderInterface;
use Neos\ContentRepository\DimensionSpace\DimensionSpace;
use Neos\ContentRepository\InMemoryGraph\Dimension\DimensionSpacePointFactory;
use Neos\ContentRepository\InMemoryGraph\ContentSubgraph\ContentSubgraphIdentifier;
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\Query\FilteredQuery as BaseElasticSearchQuery;

/**
* Query Builder for ElasticSearch Queries
*/
class ElasticSearchQuery extends BaseElasticSearchQuery
{
/**
* @Flow\Inject
* @var ContentDimensionPresetSourceInterface
*/
protected $dimensionPresetSource;

/**
* @Flow\Inject
* @var SecurityContext
*/
protected $securityContext;

/**
* @Flow\Inject
* @var DimensionSpacePointFactory
*/
protected $dimensionSpacePointFactory;

/**
* These fields are not accepted in a count request and must therefore be removed before doing so
*
* @var array
*/
protected $unsupportedFieldsInCountRequest = ['fields', 'sort', 'from', 'size', 'highlight', 'aggs', 'aggregations'];

/**
* Amount of total items in response without limit
*
* @var integer
*/
protected $totalItems;

/**
* Sets the starting point for this query. Search result should only contain nodes that
* match the context of the given node and have it as parent node in their rootline.
*
* @param NodeInterface $contextNode
* @return QueryBuilderInterface
* @api
*/
public function query(NodeInterface $contextNode)
{

// on indexing, the __parentPath is tokenized to contain ALL parent path parts,
// e.g. /foo, /foo/bar/, /foo/bar/baz; to speed up matching.. That's why we use a simple "term" filter here.
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl-term-filter.html
$this->queryFilter('term', ['__parentPath' => $contextNode->getPath()]);

$workspaceName = $contextNode->getContext()->getWorkspace()->getName();

$coordinates = [];
foreach ($contextNode->getNodeData()->getDimensionValues() as $dimensionName => $rawDimensionValues) {
$coordinates[$dimensionName] = reset($rawDimensionValues);
}
$coordinates['_workspace'] = $contextNode->getContext()->getWorkspace()->getName();

$dimensionSpacePoint = new DimensionSpace\DimensionSpacePoint($coordinates);
$contentSubgraphIdentifier = new ContentSubgraphIdentifier($workspaceName, $dimensionSpacePoint);

$edgeFilter = [
'path' => '__hierarchyRelations',
'query' => [
'bool' => [
'must' => [
[
'match' => [
'__hierarchyRelations.subgraph' => (string) $contentSubgraphIdentifier,
],
],
],
'should' => [],
'must_not' => []
]
]
];

if (!$contextNode->getContext()->isInvisibleContentShown()) {
$edgeFilter['query']['bool']['must_not'][] = [
'match' => [
'__hierarchyRelations.hidden' => true,
],
];
$edgeFilter['query']['bool']['must_not'][] = [
'range' => [
'__hierarchyRelations.hiddenBeforeDateTime' => [
'gt' => 'now'
]
]
];
$edgeFilter['query']['bool']['must_not'][] = [
'range' => [
'__hierarchyRelations.hiddenAfterDateTime' => [
'lt' => 'now'
]
]
];
}

/*
* @todo make this work
if (!$contextNode->getContext()->isInaccessibleContentShown()) {
$edgeFilter['query']['bool']['minimum_should_match'] = 1;
foreach (array_keys($this->securityContext->getRoles()) as $roleName) {
$edgeFilter['query']['bool']['should'][] = [
'term' => [
'__hierarchyRelations.accessRoles' => $roleName
],
];
}
}
*/

$this->queryFilter('nested', $edgeFilter);

$this->contextNode = $contextNode;

return $this;
}
}
Loading