Merge pull request #325 from daniellienert/feature/index-assets-2

FEATURE: Implement asset indexing using attachment-ingest plugin

Author: daniellienert

Classes/AssetExtraction/IngestAttachmentAssetExtractor.php

@@ -0,0 +1,104 @@
+<?php
+declare(strict_types=1);
+
+namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\AssetExtraction;
+
+/*
+ * This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
+ *
+ * (c) Contributors of the Neos Project - www.neos.io
+ *
+ * This package is Open Source Software. For the full copyright and license
+ * information, please view the LICENSE file which was distributed with this
+ * source code.
+ */
+
+use Neos\Flow\Annotations as FLow;
+use Neos\ContentRepository\Search\AssetExtraction\AssetExtractorInterface;
+use Neos\ContentRepository\Search\Dto\AssetContent;
+use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient;
+use Neos\Flow\Log\Utility\LogEnvironment;
+use Neos\Media\Domain\Model\AssetInterface;
+use Neos\Utility\Arrays;
+use Psr\Log\LoggerInterface;
+
+/**
+ * @Flow\Scope("singleton")
+ */
+class IngestAttachmentAssetExtractor implements AssetExtractorInterface
+{
+    /**
+     * @Flow\Inject
+     * @var ElasticSearchClient
+     */
+    protected $elasticsearchClient;
+
+    /**
+     * @Flow\Inject
+     * @var LoggerInterface
+     */
+    protected $logger;
+
+    /**
+     * Takes an asset and extracts content and meta data.
+     *
+     * @param AssetInterface $asset
+     * @return AssetContent
+     * @throws \Flowpack\ElasticSearch\Transfer\Exception
+     * @throws \Flowpack\ElasticSearch\Transfer\Exception\ApiException
+     * @throws \Neos\Flow\Http\Exception
+     */
+    public function extract(AssetInterface $asset): AssetContent
+    {
+        $request = [
+            'pipeline' => [
+                'description' => 'Attachment Extraction',
+                'processors' => [
+                    [
+                        'attachment' => [
+                            'field' => 'neos_asset',
+                            'indexed_chars' => 100000,
+                            'ignore_missing' => true,
+                        ]
+                    ]
+                ]
+            ],
+            'docs' => [
+                [
+                    '_source' => [
+                        'neos_asset' => $this->getAssetContent($asset)
+                    ]
+                ]
+            ]
+        ];
+
+        $result = $this->elasticsearchClient->request('POST', '_ingest/pipeline/_simulate', [], json_encode($request))->getTreatedContent();
+        $extractedAsset = Arrays::getValueByPath($result, 'docs.0.doc._source.attachment');
+
+        $this->logger->debug(sprintf('Extracted asset %s of type %s. Extracted %s characters of content', $asset->getResource()->getFilename(), $extractedAsset['content_type'], $extractedAsset['content_length']), LogEnvironment::fromMethodName(__METHOD__));
+
+        return new AssetContent(
+            $extractedAsset['content'] ?? '',
+            $extractedAsset['title'] ?? '',
+            $extractedAsset['name'] ?? '',
+            $extractedAsset['author'] ?? '',
+            $extractedAsset['keywords'] ?? '',
+            $extractedAsset['date'] ?? '',
+            $extractedAsset['content_type'] ?? '',
+            $extractedAsset['content_length'] ?? '',
+            $extractedAsset['language'] ?? ''
+        );
+    }
+
+    /**
+     * @param AssetInterface $asset
+     * @return null|string
+     */
+    protected function getAssetContent(AssetInterface $asset): ?string
+    {
+        $stream = $asset->getResource()->getStream();
+        stream_filter_append($stream, 'convert.base64-encode');
+        $result = stream_get_contents($stream);
+        return $result !== false ? $result : null;
+    }
+}

Configuration/Objects.yaml

@@ -4,6 +4,9 @@ Neos\ContentRepository\Search\Search\QueryBuilderInterface:
 Neos\ContentRepository\Search\Indexer\NodeIndexerInterface:
   className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer'
 
+Neos\ContentRepository\Search\AssetExtraction\AssetExtractorInterface:
+  className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\AssetExtraction\IngestAttachmentAssetExtractor'
+
 Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface:
   scope: prototype
   factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\QueryFactory'

README.md

@@ -358,7 +358,7 @@ For more information on Elasticsearch's Date Formats,
 
 ### Working with Assets / Attachments
 
-If you want to index attachments, you need to install the [Elasticsearch Attachment Plugin](https://github.com/elastic/elasticsearch-mapper-attachments).
+If you want to index attachments, you need to install the [Elasticsearch Ingest-Attachment Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest-attachment.html).
 Then, you can add the following to your `Settings.yaml`:
 
 ```yaml
@@ -368,15 +368,33 @@ Neos:
       defaultConfigurationPerType:
         'Neos\Media\Domain\Model\Asset':
           elasticSearchMapping:
-            type: attachment
-          indexing: ${Indexing.indexAsset(value)}
+            type: text
+          indexing: ${Indexing.Indexing.extractAssetContent(value)}
+```
 
-        'array<Neos\Media\Domain\Model\Asset>':
-          elasticSearchMapping:
-            type: attachment
-          indexing: ${Indexing.indexAsset(value)}
+or add the attachments content to a fulletxt field in your NodeType configuration:
+
+```yaml
+  properties:
+    file:
+      type: 'Neos\Media\Domain\Model\Asset'
+      ui:
+      search:
+        fulltextExtractor: ${Indexing.extractInto('text', Indexing.extractAssetContent(value))}
 ```
 
+By default `Indexing.extractAssetContent(value)` returns the asset content. You can use the second parameter to return asset meta data. The field parameter can be set to one of the following: `content, title, name, author, keywords, date, content_type, content_length, language`. 
+
+With that, you can for example add the keywords of a file to a higher boosted field: 
+
+```yaml
+  properties:
+    file:
+      type: 'Neos\Media\Domain\Model\Asset'
+      ui:
+      search:
+        fulltextExtractor: ${Indexing.extractInto('h2', Indexing.extractAssetContent(value, 'keywords'))}
+```
 
 
 # Query Data