MERGE: Merge branch '2.0' into master

Author: dfeyer

.travis.yml

@@ -15,6 +15,7 @@ before_install:
   - git clone https://github.com/neos/neos-base-distribution.git
   - cd neos-base-distribution
   - composer require flowpack/elasticsearch-contentrepositoryadaptor dev-master
+  - composer require typo3/typo3cr-search dev-master
 install:
   - composer install
   - cd ..

Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Eel/ElasticSearchQueryBuilder.php

@@ -14,6 +14,7 @@
 use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\QueryBuildingException;
 use TYPO3\Eel\ProtectedContextAwareInterface;
 use TYPO3\Flow\Annotations as Flow;
+use TYPO3\Flow\Utility\Arrays;
 use TYPO3\TYPO3CR\Domain\Model\NodeInterface;
 use TYPO3\TYPO3CR\Search\Search\QueryBuilderInterface;
 
@@ -101,7 +102,9 @@ class ElasticSearchQueryBuilder implements QueryBuilderInterface, ProtectedConte
                             [
                                 'match_all' => []
                             ]
-                        ]
+                        ],
+                        'should' => [],
+                        'must_not' => []
                     ]
 
                 ],
@@ -790,6 +793,28 @@ public function query(NodeInterface $contextNode)
         return $this;
     }
 
+    /**
+     * Modify a part of the Elasticsearch Request denoted by $path, merging together
+     * the existing values and the passed-in values.
+     *
+     * @param string $path
+     * @param mixed $requestPart
+     * @return $this
+     */
+    public function request($path, $requestPart)
+    {
+        $valueAtPath = Arrays::getValueByPath($this->request, $path);
+        if (is_array($valueAtPath)) {
+            $result = Arrays::arrayMergeRecursiveOverrule($valueAtPath, $requestPart);
+        } else {
+            $result = $requestPart;
+        }
+
+        $this->request = Arrays::setValueByPath($this->request, $path, $result);
+
+        return $this;
+    }
+
     /**
      * All methods are considered safe
      *

Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php

@@ -22,6 +22,7 @@
 use TYPO3\TYPO3CR\Domain\Service\ContextFactory;
 use TYPO3\TYPO3CR\Domain\Service\NodeTypeManager;
 use TYPO3\TYPO3CR\Search\Indexer\AbstractNodeIndexer;
+use TYPO3\TYPO3CR\Search\Indexer\BulkNodeIndexerInterface;
 
 /**
  * Indexer for Content Repository Nodes. Triggered from the NodeIndexingManager.
@@ -30,7 +31,7 @@
  *
  * @Flow\Scope("singleton")
  */
-class NodeIndexer extends AbstractNodeIndexer
+class NodeIndexer extends AbstractNodeIndexer implements BulkNodeIndexerInterface
 {
     /**
      * Optional postfix for the index, e.g. to have different indexes by timestamp.
@@ -88,6 +89,11 @@ class NodeIndexer extends AbstractNodeIndexer
      */
     protected $currentBulkRequest = [];
 
+    /**
+     * @var boolean
+     */
+    protected $bulkProcessing = false;
+
     /**
      * Returns the index name to be used for indexing, with optional indexNamePostfix appended.
      *
@@ -162,24 +168,26 @@ public function indexNode(NodeInterface $node, $targetWorkspaceName = null)
 
             $mappingType = $this->getIndex()->findType(NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($nodeType));
 
-            // Remove document with the same contextPathHash but different NodeType, required after NodeType change
-            $this->logger->log(sprintf('NodeIndexer: Removing node %s from index (if node type changed from %s). ID: %s', $contextPath, $node->getNodeType()->getName(), $contextPathHash), LOG_DEBUG, null, 'ElasticSearch (CR)');
-            $this->getIndex()->request('DELETE', '/_query', [], json_encode([
-                'query' => [
-                    'bool' => [
-                        'must' => [
-                            'ids' => [
-                                'values' => [$contextPathHash]
-                            ]
-                        ],
-                        'must_not' => [
-                            'term' => [
-                                '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName())
-                            ]
-                        ],
+            if ($this->bulkProcessing === false) {
+                // Remove document with the same contextPathHash but different NodeType, required after NodeType change
+                $this->logger->log(sprintf('NodeIndexer: Search and remove duplicate document if needed. ID: %s', $contextPath, $node->getNodeType()->getName(), $contextPathHash), LOG_DEBUG, null, 'ElasticSearch (CR)');
+                $this->getIndex()->request('DELETE', '/_query', [], json_encode([
+                    'query' => [
+                        'bool' => [
+                            'must' => [
+                                'ids' => [
+                                    'values' => [$contextPathHash]
+                                ]
+                            ],
+                            'must_not' => [
+                                'term' => [
+                                    '_type' => NodeTypeMappingBuilder::convertNodeTypeNameToMappingName($node->getNodeType()->getName())
+                                ]
+                            ],
+                        ]
                     ]
-                ]
-            ]));
+                ]));
+            }
 
             if ($node->isRemoved()) {
                 // TODO: handle deletion from the fulltext index as well
@@ -541,4 +549,26 @@ public function removeOldIndices()
 
         return $indicesToBeRemoved;
     }
+
+    /**
+     * Perform indexing without checking about duplication document
+     *
+     * This is used during bulk indexing to improve performance
+     *
+     * @param callable $callback
+     * @throws \Exception
+     */
+    public function withBulkProcessing(callable $callback)
+    {
+        $bulkProcessing = $this->bulkProcessing;
+        $this->bulkProcessing = true;
+        try {
+            /** @noinspection PhpUndefinedMethodInspection */
+            $callback->__invoke();
+        } catch (\Exception $exception) {
+            $this->bulkProcessing = $bulkProcessing;
+            throw $exception;
+        }
+        $this->bulkProcessing = $bulkProcessing;
+    }
 }

Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Service/IndexWorkspaceTrait.php

@@ -84,6 +84,9 @@ protected function indexWorkspaceWithDimensions($workspaceName, array $dimension
 
         $traverseNodes($rootNode, $indexedNodes);
 
+        $this->nodeFactory->reset();
+        $context->getFirstLevelNodeCache()->flush();
+         
         if ($callback !== null) {
             $callback($workspaceName, $indexedNodes, $dimensions);
         }

README.md

@@ -224,6 +224,11 @@ Furthermore, there is a more low-level operator which can be used to add arbitra
 
 * `queryFilter("filterType", {option1: "value1"})`
 
+At lowest level, there is the `request` operator which allows to modify the request in arbitrary manner. Note that the existing request is merged with the passed-in type in case it is an array:
+
+* `request('query.filtered.query.bool.minimum_should_match', 1)`
+* `request('query.filtered.query.bool', {"minimum_should_match": 1})`
+
 In order to debug the query more easily, the following operation is helpful:
 
 * `log()` log the full query on execution into the Elasticsearch log (i.e. in `Data/Logs/ElasticSearch.log`)

Tests/Unit/Eel/ElasticSearchQueryBuilderTest.php

@@ -55,7 +55,9 @@ public function basicRequestStructureTakesContextNodeIntoAccount()
                         'bool' => [
                             'must' => [
                                 ['match_all' => []]
-                            ]
+                            ],
+                            'should' => [],
+                            'must_not' => []
                         ]
                     ],
                     'filter' => [
@@ -326,6 +328,32 @@ public function aCustomAggregationDefinitionCanBeApplied()
         $this->assertInArray($expected, $actual);
     }
 
+    /**
+     * @test
+     */
+    public function requestCanBeExtendedByArbitraryProperties()
+    {
+        $this->queryBuilder->request('foo.bar', ['field' => 'x']);
+        $expected = [
+            'bar' => ['field' => 'x']
+        ];
+        $actual = $this->queryBuilder->getRequest();
+        $this->assertEquals($expected, $actual['foo']);
+    }
+
+    /**
+     * @test
+     */
+    public function existingRequestPropertiesCanBeOverridden()
+    {
+        $this->queryBuilder->limit(2);
+        $this->queryBuilder->request('limit', 10);
+        $expected = 10;
+        $actual = $this->queryBuilder->getRequest();
+        $this->assertEquals($expected, $actual['limit']);
+    }
+
+
     /**
      * Test helper
      *