Skip to content

Commit b86c930

Browse files
dfeyerdfeyer
committed
TASK: Use the ErrorHandlingService during bulk indexing
1 parent 55d3499 commit b86c930

4 files changed

Lines changed: 213 additions & 2 deletions

File tree

‎Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/Error/BulkIndexingError.php‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\Error;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface;
15+
use TYPO3\Flow\Annotations as Flow;
16+
use TYPO3\Flow\Exception;
17+
18+
/**
19+
* Handle Bulk Indexing Error and build human readable output for analysis
20+
*/
21+
class BulkIndexingError implements ErrorInterface
22+
{
23+
/**
24+
* @Flow\Inject
25+
* @var LoggerInterface
26+
*/
27+
protected $logger;
28+
29+
/**
30+
* @var string
31+
*/
32+
protected $message;
33+
34+
/**
35+
* @var string
36+
*/
37+
protected $filename;
38+
39+
/**
40+
* @var array
41+
*/
42+
protected $currentBulkRequest;
43+
44+
/**
45+
* @var array
46+
*/
47+
protected $errors;
48+
49+
/**
50+
* @param array $currentBulkRequest
51+
* @param array $errors
52+
* @throws Exception
53+
*/
54+
public function __construct(array $currentBulkRequest, $errors)
55+
{
56+
$this->currentBulkRequest = $currentBulkRequest;
57+
$this->errors = json_decode($errors, true);
58+
59+
if (!file_exists(FLOW_PATH_DATA . 'Logs/ElasticSearch')) {
60+
mkdir(FLOW_PATH_DATA . 'Logs/ElasticSearch');
61+
}
62+
63+
$referenceCode = date('YmdHis', $_SERVER['REQUEST_TIME']) . substr(md5(rand()), 0, 6);
64+
65+
$this->filename = FLOW_PATH_DATA . 'Logs/ElasticSearch/' . $referenceCode . '.txt';
66+
$this->message = sprintf('Bulk indexing errors detected - See also: Data/Logs/ElasticSearch/%s on host: %s', basename($this->filename), gethostname());
67+
}
68+
69+
/**
70+
* Log the error message
71+
*/
72+
public function log()
73+
{
74+
if (file_exists(FLOW_PATH_DATA . 'Logs/ElasticSearch') && is_dir(FLOW_PATH_DATA . 'Logs/ElasticSearch') && is_writable(FLOW_PATH_DATA . 'Logs/ElasticSearch')) {
75+
file_put_contents($this->filename, $this->renderErrors());
76+
$this->logger->log($this->message, LOG_ERR, [], 'Flowpack.ElasticSearch.ContentRepositoryAdaptor', __CLASS__, __FUNCTION__);
77+
} else {
78+
$this->logger->log(sprintf('Could not write indexing errors backtrace into %s because the directory could not be created or is not writable.', FLOW_PATH_DATA . 'Logs/ElasticSearch/'), LOG_WARNING, [], 'Flowpack.ElasticSearch.ContentRepositoryAdaptor', __CLASS__, __FUNCTION__);
79+
}
80+
}
81+
82+
/**
83+
* @return string
84+
*/
85+
public function message()
86+
{
87+
return $this->message;
88+
}
89+
90+
91+
/**
92+
* @return string
93+
*/
94+
protected function renderErrors()
95+
{
96+
$bulkRequest = json_encode($this->currentBulkRequest, JSON_PRETTY_PRINT);
97+
$errors = json_encode($this->errors, JSON_PRETTY_PRINT);
98+
return sprintf("Payload:\n========\n\n%s\n\nErrors:\n=======\n\n%s\n\n", $bulkRequest, $errors);
99+
}
100+
}

‎Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/Error/ErrorInterface.php‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\Error;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use TYPO3\Flow\Annotations as Flow;
15+
16+
/**
17+
* Error Interface
18+
*/
19+
interface ErrorInterface
20+
{
21+
/**
22+
* Log the error message
23+
*/
24+
public function log();
25+
26+
/**
27+
* Get a short log message for reporting
28+
*
29+
* @return string
30+
*/
31+
public function message();
32+
}

‎Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/Error/MalformedBulkRequestError.php‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<?php
2+
namespace Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\Error;
3+
4+
/*
5+
* This file is part of the Flowpack.ElasticSearch.ContentRepositoryAdaptor package.
6+
*
7+
* (c) Contributors of the Neos Project - www.neos.io
8+
*
9+
* This package is Open Source Software. For the full copyright and license
10+
* information, please view the LICENSE file which was distributed with this
11+
* source code.
12+
*/
13+
14+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\LoggerInterface;
15+
use TYPO3\Flow\Annotations as Flow;
16+
17+
/**
18+
* Handle malformed bulk request error
19+
*/
20+
class MalformedBulkRequestError implements ErrorInterface
21+
{
22+
/**
23+
* @Flow\Inject
24+
* @var LoggerInterface
25+
*/
26+
protected $logger;
27+
28+
/**
29+
* @var string
30+
*/
31+
protected $message;
32+
33+
/**
34+
* @var array
35+
*/
36+
protected $tuple;
37+
38+
/**
39+
* MalformedBulkRequestError constructor.
40+
* @param string $message
41+
* @param array $tuple
42+
*/
43+
public function __construct($message, array $tuple)
44+
{
45+
$this->message = $message;
46+
$this->tuple = $tuple;
47+
}
48+
49+
/**
50+
* Log the error message
51+
*/
52+
public function log()
53+
{
54+
$this->logger->log($this->message(), LOG_ERR, $this->tuple);
55+
}
56+
57+
/**
58+
* Get a short log message for reporting
59+
*
60+
* @return string
61+
*/
62+
public function message()
63+
{
64+
return $this->message;
65+
}
66+
}

‎Classes/Flowpack/ElasticSearch/ContentRepositoryAdaptor/Indexer/NodeIndexer.php‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313

1414
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient;
1515
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception;
16+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\Error\BulkIndexingError;
17+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\Error\MalformedBulkRequestError;
1618
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Mapping\NodeTypeMappingBuilder;
19+
use Flowpack\ElasticSearch\ContentRepositoryAdaptor\Service\ErrorHandlingService;
1720
use Flowpack\ElasticSearch\Domain\Model\Document as ElasticSearchDocument;
1821
use Flowpack\ElasticSearch\Domain\Model\Index;
1922
use TYPO3\Flow\Annotations as Flow;
@@ -40,6 +43,12 @@ class NodeIndexer extends AbstractNodeIndexer implements BulkNodeIndexerInterfac
4043
*/
4144
protected $indexNamePostfix = '';
4245

46+
/**
47+
* @Flow\Inject
48+
* @var ErrorHandlingService
49+
*/
50+
protected $errorHandlingService;
51+
4352
/**
4453
* @Flow\Inject
4554
* @var ElasticSearchClient
@@ -427,7 +436,9 @@ public function flush()
427436
foreach ($bulkRequestTuple as $bulkRequestItem) {
428437
$itemAsJson = json_encode($bulkRequestItem);
429438
if ($itemAsJson === false) {
430-
$this->logger->log('Indexing Error: Bulk request item could not be encoded as JSON - ' . json_last_error_msg(), LOG_ERR, $bulkRequestItem);
439+
$this->errorHandlingService->log(
440+
new MalformedBulkRequestError('Indexing Error: Bulk request item could not be encoded as JSON - ' . json_last_error_msg(), $bulkRequestItem)
441+
);
431442
continue 2;
432443
}
433444
$tupleAsJson .= $itemAsJson . chr(10);
@@ -440,7 +451,9 @@ public function flush()
440451
foreach (explode("\n", $responseAsLines) as $responseLine) {
441452
$response = json_decode($responseLine);
442453
if (!is_object($response) || (isset($response->errors) && $response->errors !== false)) {
443-
$this->logger->log('Indexing Error: ' . $responseLine, LOG_ERR);
454+
$this->errorHandlingService->log(
455+
new BulkIndexingError($this->currentBulkRequest, $responseLine)
456+
);
444457
}
445458
}
446459
}

0 commit comments

Comments
 (0)