Merge pull request #108 from Flowpack/tast-better-dimensions

[TASK] Better support for Dimensions

This change take care of Node variants in all dimension combinations
during Indexing. This change introduce a new property in the Elasticsearch index called __dimensionCombinationHash so you need to reindex all your nodes.

Author: daniellienert

.travis.yml

@@ -22,4 +22,4 @@ install:
   - cd neos-base-distribution
 script:
   - bin/phpunit -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit
-  - FLOW_CONTEXT=Testing/WithoutLanguageFallback bin/phpunit -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional
+  - bin/phpunit -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional

Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Command/NodeIndexCommandController.php

@@ -14,6 +14,7 @@
 use TYPO3\Flow\Annotations as Flow;
 use TYPO3\Flow\Cli\CommandController;
 use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder;
+use TYPO3\TYPO3CR\Domain\Service\ContentDimensionCombinator;
 use TYPO3\TYPO3CR\Search\Indexer\NodeIndexingManager;
 
 /**
@@ -98,6 +99,12 @@ class NodeIndexCommandController extends CommandController
      */
     protected $configurationManager;
 
+    /**
+     * @Flow\Inject
+     * @var ContentDimensionCombinator
+     */
+    protected $contentDimensionCombinator;
+
     /**
      * @var array
      */
@@ -158,14 +165,19 @@ public function showMappingCommand()
      * @param integer $limit Amount of nodes to index at maximum
      * @param boolean $update if TRUE, do not throw away the index at the start. Should *only be used for development*.
      * @param string $workspace name of the workspace which should be indexed
+     * @param string $postfix Index postfix, index with the same postifix will be deleted if exist
      * @return void
      */
-    public function buildCommand($limit = null, $update = false, $workspace = null)
+    public function buildCommand($limit = null, $update = false, $workspace = null, $postfix = null)
     {
         if ($update === true) {
             $this->logger->log('!!! Update Mode (Development) active!', LOG_INFO);
         } else {
-            $this->nodeIndexer->setIndexNamePostfix(time());
+            $this->nodeIndexer->setIndexNamePostfix($postfix ?: time());
+            if ($this->nodeIndexer->getIndex()->exists() === true) {
+                $this->logger->log(sprintf('Deleted index with the same postfix (%s)!', $postfix), LOG_WARNING);
+                $this->nodeIndexer->getIndex()->delete();
+            }
             $this->nodeIndexer->getIndex()->create();
             $this->logger->log('Created index ' . $this->nodeIndexer->getIndexName(), LOG_INFO);
 
@@ -238,7 +250,7 @@ public function cleanupCommand()
      */
     protected function indexWorkspace($workspaceName)
     {
-        $combinations = $this->calculateDimensionCombinations();
+        $combinations = $this->contentDimensionCombinator->getAllAllowedCombinations();
         if ($combinations === array()) {
             $this->indexWorkspaceWithDimensions($workspaceName);
         } else {
@@ -287,41 +299,4 @@ protected function traverseNodes(\TYPO3\TYPO3CR\Domain\Model\NodeInterface $curr
             $this->traverseNodes($childNode);
         }
     }
-
-    /**
-     * @return array
-     * @todo will went into TYPO3CR
-     */
-    protected function calculateDimensionCombinations()
-    {
-        $dimensionPresets = $this->contentDimensionPresetSource->getAllPresets();
-
-        $dimensionValueCountByDimension = array();
-        $possibleCombinationCount = 1;
-        $combinations = array();
-
-        foreach ($dimensionPresets as $dimensionName => $dimensionPreset) {
-            if (isset($dimensionPreset['presets']) && !empty($dimensionPreset['presets'])) {
-                $dimensionValueCountByDimension[$dimensionName] = count($dimensionPreset['presets']);
-                $possibleCombinationCount = $possibleCombinationCount * $dimensionValueCountByDimension[$dimensionName];
-            }
-        }
-
-        foreach ($dimensionPresets as $dimensionName => $dimensionPreset) {
-            for ($i = 0; $i < $possibleCombinationCount; $i++) {
-                if (!isset($combinations[$i]) || !is_array($combinations[$i])) {
-                    $combinations[$i] = array();
-                }
-
-                $currentDimensionCurrentPreset = current($dimensionPresets[$dimensionName]['presets']);
-                $combinations[$i][$dimensionName] = $currentDimensionCurrentPreset['values'];
-
-                if (!next($dimensionPresets[$dimensionName]['presets'])) {
-                    reset($dimensionPresets[$dimensionName]['presets']);
-                }
-            }
-        }
-
-        return $combinations;
-    }
 }

Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php

@@ -650,14 +650,13 @@ public function fetch()
         $this->result = $response->getTreatedContent();
 
         $this->result['nodes'] = array();
-        if (array_key_exists('hits', $this->result) && is_array($this->result['hits']) && count($this->result['hits']) > 0) {
-            $this->result['nodes'] = $this->convertHitsToNodes($this->result['hits']);
-        }
-
         if ($this->logThisQuery === true) {
             $this->logger->log(sprintf('Query Log (%s): %s -- execution time: %s ms -- Limit: %s -- Number of results returned: %s -- Total Results: %s',
                 $this->logMessage, json_encode($this->request), (($timeAfterwards - $timeBefore) * 1000), $this->limit, count($this->result['hits']['hits']), $this->result['hits']['total']), LOG_DEBUG);
         }
+        if (array_key_exists('hits', $this->result) && is_array($this->result['hits']) && count($this->result['hits']) > 0) {
+            $this->result['nodes'] = $this->convertHitsToNodes($this->result['hits']);
+        }
 
         return $this->result;
     }
@@ -699,7 +698,7 @@ public function count()
         $count = $treatedContent['count'];
 
         if ($this->logThisQuery === true) {
-            $this->logger->log('Query Log (' . $this->logMessage . '): ' . json_encode($this->request) . ' -- execution time: ' . (($timeAfterwards - $timeBefore) * 1000) . ' ms -- Total Results: ' . $count, LOG_DEBUG);
+            $this->logger->log('Count Query Log (' . $this->logMessage . '): ' . json_encode($this->request) . ' -- execution time: ' . (($timeAfterwards - $timeBefore) * 1000) . ' ms -- Total Results: ' . $count, LOG_DEBUG);
         }
 
         return $count;
@@ -773,8 +772,9 @@ public function query(NodeInterface $contextNode)
         $this->queryFilter('terms', array('__workspace' => array_unique(array('live', $contextNode->getContext()->getWorkspace()->getName()))));
 
         // match exact dimension values for each dimension, this works because the indexing flattens the node variants for all dimension preset combinations
-        foreach ($contextNode->getContext()->getDimensions() as $dimensionName => $dimensionValues) {
-            $this->queryFilter('terms', array('__dimensionCombinations.' . $dimensionName => $dimensionValues, 'execution' => 'and'));
+        $dimensionCombinations = $contextNode->getContext()->getDimensions();
+        if (is_array($dimensionCombinations)) {
+            $this->queryFilter('term', ['__dimensionCombinationHash' => md5(json_encode($dimensionCombinations))]);
         }
 
         $this->contextNode = $contextNode;
@@ -817,15 +817,9 @@ protected function convertHitsToNodes(array $hits)
          * we might be able to use https://github.com/elasticsearch/elasticsearch/issues/3300 as soon as it is merged.
          */
         foreach ($hits['hits'] as $hit) {
-            // with ElasticSearch 1.0 fields always returns an array,
-            // see https://github.com/Flowpack/Flowpack.ElasticSearch.ContentRepositoryAdaptor/issues/17
-            if (is_array($hit['fields']['__path'])) {
-                $nodePath = current($hit['fields']['__path']);
-            } else {
-                $nodePath = $hit['fields']['__path'];
-            }
+            $nodePath = current($hit['fields']['__path']);
             $node = $this->contextNode->getNode($nodePath);
-            if ($node instanceof NodeInterface) {
+            if ($node instanceof NodeInterface && !isset($nodes[$node->getIdentifier()])) {
                 $nodes[$node->getIdentifier()] = $node;
                 $elasticSearchHitPerNode[$node->getIdentifier()] = $hit;
                 if ($this->limit > 0 && count($nodes) >= $this->limit) {
@@ -835,7 +829,7 @@ protected function convertHitsToNodes(array $hits)
         }
 
         if ($this->logThisQuery === true) {
-            $this->logger->log('Query Log (' . $this->logMessage . ') Number of returned results: ' . count($nodes), LOG_DEBUG);
+            $this->logger->log('Returned nodes (' . $this->logMessage . '): ' . count($nodes), LOG_DEBUG);
         }
 
         $this->elasticSearchHitsIndexedByNodeFromLastRequest = $elasticSearchHitPerNode;