Merge branch 'master' into fulltext-search-with-simple-query-string

Author: skurfuerst

.travis.yml

@@ -9,9 +9,9 @@ addons:
       - openjdk-8-jre-headless
 matrix:
   include:
-    - php: 7.2
+    - php: 7.3
       env: ES=6
-    - php: 7.2
+    - php: 7.3
       env: ES=7
     - php: 7.4
       env: ES=6
@@ -23,7 +23,7 @@ cache:
     - $HOME/.composer/cache
 
 before_install:
-  - export NEOS_TARGET_VERSION=5.3
+  - export NEOS_TARGET_VERSION=7.0
   - cd ..
   - if [ "$ES" = 6 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && tar xvfz elasticsearch-6.8.6.tar.gz && mv elasticsearch-6.8.6 elasticsearch; fi
   - if [ "$ES" = 7 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.0-linux-x86_64.tar.gz && tar xvfz elasticsearch-7.9.0-linux-x86_64.tar.gz && mv elasticsearch-7.9.0 elasticsearch; fi
@@ -35,7 +35,6 @@ before_install:
   - composer require --no-update --no-interaction neos/content-repository-search:dev-master
   - composer require --no-update --no-interaction flowpack/elasticsearch:dev-master
   - composer require --no-update --no-interaction flowpack/elasticsearch-contentrepositoryadaptor:dev-master
-  - composer remove --no-update --no-interaction neos/site-kickstarter
 install:
   - composer install --no-interaction
   - cd ..

Classes/AssetExtraction/IngestAttachmentAssetExtractor.php

@@ -13,10 +13,11 @@
  * source code.
  */
 
-use Neos\Flow\Annotations as FLow;
+use Neos\Flow\Annotations as Flow;
 use Neos\ContentRepository\Search\AssetExtraction\AssetExtractorInterface;
 use Neos\ContentRepository\Search\Dto\AssetContent;
 use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient;
+use Neos\Flow\Log\ThrowableStorageInterface;
 use Neos\Flow\Log\Utility\LogEnvironment;
 use Neos\Media\Domain\Model\AssetInterface;
 use Neos\Utility\Arrays;
@@ -33,12 +34,24 @@ class IngestAttachmentAssetExtractor implements AssetExtractorInterface
      */
     protected $elasticsearchClient;
 
+    /**
+     * @Flow\Inject
+     * @var ThrowableStorageInterface
+     */
+    protected $throwableStorage;
+
     /**
      * @Flow\Inject
      * @var LoggerInterface
      */
     protected $logger;
 
+    /**
+     * @Flow\InjectConfiguration(package="Flowpack.ElasticSearch.ContentRepositoryAdaptor", path="indexing.assetExtraction.maximumFileSize")
+     * @var int
+     */
+    protected $maximumFileSize;
+
     /**
      * Takes an asset and extracts content and meta data.
      *
@@ -50,6 +63,13 @@ class IngestAttachmentAssetExtractor implements AssetExtractorInterface
      */
     public function extract(AssetInterface $asset): AssetContent
     {
+        if ($asset->getResource()->getFileSize() > $this->maximumFileSize) {
+            $this->logger->info(sprintf('The asset %s with size of %s bytes exceeds the maximum size of %s bytes. The file content was not ingested.', $asset->getResource()->getFilename(), $asset->getResource()->getFileSize(), $this->maximumFileSize), LogEnvironment::fromMethodName(__METHOD__));
+            return $this->buildAssetContentObject([]);
+        }
+
+        $extractedAsset = null;
+
         $request = [
             'pipeline' => [
                 'description' => 'Attachment Extraction',
@@ -73,14 +93,50 @@ public function extract(AssetInterface $asset): AssetContent
         ];
 
         $result = $this->elasticsearchClient->request('POST', '_ingest/pipeline/_simulate', [], json_encode($request))->getTreatedContent();
-        $extractedAsset = Arrays::getValueByPath($result, 'docs.0.doc._source.attachment');
+
+        if (is_array($result)) {
+            $extractedAsset = Arrays::getValueByPath($result, 'docs.0.doc._source.attachment');
+        }
 
         if (!is_array($extractedAsset)) {
-            $this->logger->error(sprintf('Error while extracting fulltext data from file "%s". See Elasticsearch error log line fo details.', $asset->getResource()->getFilename()), LogEnvironment::fromMethodName(__METHOD__));
+            $this->logger->error(sprintf('Error while extracting fulltext data from file "%s". See Elasticsearch error log line for details.', $asset->getResource()->getFilename()), LogEnvironment::fromMethodName(__METHOD__));
         } else {
             $this->logger->debug(sprintf('Extracted asset %s of type %s. Extracted %s characters of content', $asset->getResource()->getFilename(), $extractedAsset['content_type'] ?? '-no-content-type-', $extractedAsset['content_length'] ?? '0'), LogEnvironment::fromMethodName(__METHOD__));
         }
 
+        return $this->buildAssetContentObject($extractedAsset);
+    }
+
+    /**
+     * @param AssetInterface $asset
+     * @return string
+     */
+    protected function getAssetContent(AssetInterface $asset): string
+    {
+        try {
+            $stream = $asset->getResource()->getStream();
+        } catch (\Exception $e) {
+            $message = $this->throwableStorage->logThrowable($e);
+            $this->logger->error(sprintf('An exception occured while fetching resource with sha1 %s of asset %s. %s', $asset->getResource()->getSha1(), $asset->getResource()->getFilename(), $message), LogEnvironment::fromMethodName(__METHOD__));
+            return '';
+        }
+
+        if ($stream === false) {
+            $this->logger->error(sprintf('Could not get the file stream of resource with sha1 %s of asset %s.', $asset->getResource()->getSha1(), $asset->getResource()->getFilename()), LogEnvironment::fromMethodName(__METHOD__));
+            return '';
+        }
+
+        stream_filter_append($stream, 'convert.base64-encode');
+        $result = stream_get_contents($stream);
+        return $result !== false ? $result : '';
+    }
+
+    /**
+     * @param $extractedAsset
+     * @return AssetContent
+     */
+    protected function buildAssetContentObject(?array $extractedAsset): AssetContent
+    {
         return new AssetContent(
             $extractedAsset['content'] ?? '',
             $extractedAsset['title'] ?? '',
@@ -93,16 +149,4 @@ public function extract(AssetInterface $asset): AssetContent
             $extractedAsset['language'] ?? ''
         );
     }
-
-    /**
-     * @param AssetInterface $asset
-     * @return null|string
-     */
-    protected function getAssetContent(AssetInterface $asset): ?string
-    {
-        $stream = $asset->getResource()->getStream();
-        stream_filter_append($stream, 'convert.base64-encode');
-        $result = stream_get_contents($stream);
-        return $result !== false ? $result : null;
-    }
 }