From 6e22457563a3c241a5fec7f728a66919256dbca3 Mon Sep 17 00:00:00 2001 From: Ludovic <54670129+lbr38@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:16:57 +0200 Subject: [PATCH] 5.14.0 --- www/config/service-units.php | 10 ++ www/controllers/App/Session.php | 29 +++- www/controllers/Filesystem/Directory.php | 2 +- www/controllers/HttpRequest.php | 3 + .../Container/vars/browse/list.vars.inc.php | 6 +- www/controllers/Layout/Layout.php | 6 +- www/controllers/Repo/Metadata/Rpm.php | 159 +++++++++--------- www/controllers/Repo/Mirror/Rpm.php | 13 +- www/controllers/Repo/Snapshot/Package.php | 7 + www/controllers/Repo/Snapshot/Snapshot.php | 21 ++- www/controllers/Repo/Task/Duplicate.php | 5 + www/controllers/Repo/Task/Finalize.php | 6 + www/controllers/Service/Unit/Main.php | 33 ++-- www/controllers/Service/Unit/Repo/Size.php | 49 ++++++ www/controllers/User/Edit.php | 9 +- www/controllers/User/Login.php | 154 ++++++----------- www/controllers/ajax/repo/get.php | 21 --- www/models/Connection.php | 5 +- www/models/Repo/Listing.php | 7 + www/models/Repo/Repo.php | 8 + www/models/Repo/Snapshot/Snapshot.php | 42 ++++- www/public/resources/js/classes/Repo.js | 32 ---- .../resources/js/events/repo/install.js | 2 +- www/public/resources/js/functions/browse.js | 2 + .../source-repositories/rpm/centos.yml | 10 +- www/tools/service.php | 8 +- www/update/database/5.14.0.php | 36 ++++ www/version | 2 +- www/views/includes/repos-list.inc.php | 15 +- 29 files changed, 400 insertions(+), 302 deletions(-) create mode 100644 www/controllers/Service/Unit/Repo/Size.php create mode 100644 www/update/database/5.14.0.php diff --git a/www/config/service-units.php b/www/config/service-units.php index f52a434b8..425a51bd5 100644 --- a/www/config/service-units.php +++ b/www/config/service-units.php @@ -52,6 +52,16 @@ 'frequency' => 'every-minute', 'log-dir' => 'system/monitoring' ], + // This calculate the size of repository snapshots + 'snapshots-size-calculation' => [ + 'title' => 'Snapshots size calculation', + 'description' => 'Re-calculates the size of repository snapshots and stores it in the database', + 'controller' => 'Service\Unit\Repo\Size', + 'method' => 'calculate', + 'frequency' => 'every-day', + 'time' => '00:00', + 'log-dir' => 'repos/size' + ], // This generates statistics of repositories (size, packages count) every day at midnight 'stats-generate' => [ 'title' => 'Repositories statistics generation', diff --git a/www/controllers/App/Session.php b/www/controllers/App/Session.php index ce53c2f92..8df218166 100644 --- a/www/controllers/App/Session.php +++ b/www/controllers/App/Session.php @@ -2,8 +2,6 @@ namespace Controllers\App; -use Exception; - class Session { /** @@ -14,7 +12,7 @@ public static function load() /** * Start session */ - if (!isset($_SESSION)) { + if (session_status() !== PHP_SESSION_ACTIVE) { session_start([ 'cookie_secure' => (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on'), 'cookie_httponly' => true, @@ -41,5 +39,30 @@ public static function load() * Define the new session start time (or renew the current session) */ $_SESSION['start_time'] = time(); + + /** + * Release the session file lock immediately + * Otherwise PHP keeps an exclusive lock on the session file for the whole request duration, which serializes all the concurrent + * requests of the same user (e.g. slow ajax calls would block every other action of the interface) + * Any code that needs to persist new session variables must use Session::set() + */ + session_write_close(); + } + + /** + * Update session variables + * The session is closed right after being loaded, so it must be reopened to persist any change + */ + public static function set(array $variables) : void + { + if (session_status() !== PHP_SESSION_ACTIVE) { + session_start(); + } + + foreach ($variables as $name => $value) { + $_SESSION[$name] = $value; + } + + session_write_close(); } } diff --git a/www/controllers/Filesystem/Directory.php b/www/controllers/Filesystem/Directory.php index fc3dedcc2..0852a7f1d 100644 --- a/www/controllers/Filesystem/Directory.php +++ b/www/controllers/Filesystem/Directory.php @@ -165,7 +165,7 @@ public static function deleteIfEmpty(array $directories) : void /** * Get directory size in bytes */ - public static function getSize(string $path) + public static function getSize(string $path): string { $bytestotal = 0; $path = realpath($path); diff --git a/www/controllers/HttpRequest.php b/www/controllers/HttpRequest.php index 56ab6ce1b..451728d6b 100644 --- a/www/controllers/HttpRequest.php +++ b/www/controllers/HttpRequest.php @@ -128,6 +128,9 @@ public function get(array $params, bool $parseJson = false, string $jsonExtract curl_setopt($this->ch, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); } + // Add a custom user agent to avoid being blocked by some servers + curl_setopt($this->ch, CURLOPT_USERAGENT, 'Repomanager (+https://github.com/lbr38/repomanager)'); + // If the target must be saved as a file directly if ($saveToFile) { if ($headRequest) { diff --git a/www/controllers/Layout/Container/vars/browse/list.vars.inc.php b/www/controllers/Layout/Container/vars/browse/list.vars.inc.php index c551b50e9..5c63fc9b0 100644 --- a/www/controllers/Layout/Container/vars/browse/list.vars.inc.php +++ b/www/controllers/Layout/Container/vars/browse/list.vars.inc.php @@ -46,17 +46,15 @@ * Retrieve repo size and packages count */ if ($myrepo->getPackageType() == 'rpm') { - $repoSize = \Controllers\Filesystem\Directory::getSize($snapshotPath); $packagesCount = count(\Controllers\Filesystem\File::findRecursive($snapshotPath, [], ['rpm'])); } if ($myrepo->getPackageType() == 'deb') { - $repoSize = \Controllers\Filesystem\Directory::getSize($snapshotPath); $packagesCount = count(\Controllers\Filesystem\File::findRecursive($snapshotPath, [], ['deb'])); } /** - * Convert repo size in the most suitable byte format + * Get the size of the snapshot */ -$repoSize = \Controllers\Utils\Convert::sizeToHuman($repoSize); +$repoSize = $repoSnapshotController->getSize($snapId, true) ?? 'Unknown'; unset($repoSnapshotController); diff --git a/www/controllers/Layout/Layout.php b/www/controllers/Layout/Layout.php index 87fbd1e86..c57aa3d94 100644 --- a/www/controllers/Layout/Layout.php +++ b/www/controllers/Layout/Layout.php @@ -45,8 +45,10 @@ private function login() */ private function logout() { - // Start the session - session_start(); + // Start the session if not already active + if (session_status() !== PHP_SESSION_ACTIVE) { + session_start(); + } // Reinitialize the session array, empty it $_SESSION = []; diff --git a/www/controllers/Repo/Metadata/Rpm.php b/www/controllers/Repo/Metadata/Rpm.php index a0c542e6a..c7f48654a 100644 --- a/www/controllers/Repo/Metadata/Rpm.php +++ b/www/controllers/Repo/Metadata/Rpm.php @@ -12,118 +12,115 @@ class Rpm extends Metadata private $createrepoArgs = '-v --compress-type=gz --general-compress-type=gz'; private $modifyrepo = '/usr/bin/modifyrepo_c'; private $modifyrepoArgs = '--compress-type=gz'; - - public function setRoot(string $root) + private $additionalMetadataFiles = [ + 'updateinfo.xml.gz' => [ + 'mdtype' => 'updateinfo', + 'temp' => 'tmp-updateinfo' + ], + 'comps.xml' => [ + 'mdtype' => 'group', + 'temp' => 'tmp-comps' + ], + 'modules.yaml' => [ + 'mdtype' => 'modules', + 'temp' => 'tmp-modules' + ] + ]; + + public function setRoot(string $root): void { $this->root = $root; } /** - * Add an additional metadata file to the repository metadata using modifyrepo_c - * Searches for the file with various compressions and deletes it afterwards + * Create metadata files + * Use createrepo_c to generate repository metadata + * Use modifyrepo_c to add additional metadata files to the repository metadata */ - private function addMetadata(string $filePrefix, string $mdtype): void + public function create(): void { - // Check if modifyrepo_c is available - if (!file_exists($this->modifyrepo)) { - throw new Exception('Could not find modifyrepo_c on the system'); - } + $this->taskLogSubStepController->new('create-metadata', 'GENERATING REPOSITORY METADATA'); - // Look for the metadata file with various compression formats (and plain format) and use the first one found - $metadataFile = null; - foreach ([$filePrefix, $filePrefix . '.gz', $filePrefix . '.xz', $filePrefix . '.bz2', $filePrefix . '.zst'] as $file) { - if (file_exists($this->root . '/' . $file)) { - $metadataFile = $file; - break; + // Check if createrepo_c and modifyrepo_c exist on the system + foreach ([$this->createrepo, $this->modifyrepo] as $bin) { + if (!file_exists($bin)) { + throw new Exception('Could not find ' . $bin . ' on the system'); } } - // If file doesn't exist, return silently (file may not exist in the source repo) - if (empty($metadataFile)) { - return; + // Check if root path exists + if (!is_dir($this->root)) { + throw new Exception('Repository root directory ' . $this->root . ' does not exist'); } - // Add the file to the repository metadata using modifyrepo_c - $modifyrepoProcess = new Process($this->modifyrepo . ' ' . $this->modifyrepoArgs . ' --mdtype=' . $mdtype . ' ' . $this->root . '/' . $metadataFile . ' ' . $this->root . '/repodata/'); - $modifyrepoProcess->setBackground(true); - $modifyrepoProcess->execute(); - /** - * Retrieve PID of the launched process - * Then write PID to main PID file + * Rename additional metadata files to temporary names to avoid them being included automatically by createrepo_c + * They will be added later using modifyrepo_c + * This to avoid issues with broken modules.yaml (from Oracle 8 Appstream repo notably) raising errors in createrepo_c (modifyrepo_c does not have this issue) + * https://github.com/lbr38/repomanager/issues/399 + * https://github.com/lbr38/repomanager/issues/408 */ - $this->taskController->addsubpid($modifyrepoProcess->getPid()); - - // Retrieve output from process - $this->taskLogSubStepController->output($modifyrepoProcess->getOutput(), 'pre'); - - if ($modifyrepoProcess->getExitCode() != 0) { - throw new Exception('Could not add ' . $mdtype . ' to repository metadata'); + foreach ($this->additionalMetadataFiles as $file => $value) { + if (file_exists($this->root . '/' . $file)) { + if (!rename($this->root . '/' . $file, $this->root . '/' . $value['temp'])) { + throw new Exception('Could not rename ' . $this->root . '/' . $file . ' to ' . $this->root . '/' . $value['temp']); + } + } } - $modifyrepoProcess->close(); + // Launch createrepo_c to generate repository metadata + $process = new Process($this->createrepo . ' ' . $this->createrepoArgs . ' ' . $this->root . '/'); + $process->setBackground(true); + $process->execute(); - // Delete the file as it's now part of the metadata - if (!unlink($this->root . '/' . $metadataFile)) { - throw new Exception('Could not delete ' . $this->root . '/' . $metadataFile); - } - } + // Add PID to main PID file + $this->taskController->addsubpid($process->getPid()); - /** - * Create metadata files - */ - public function create(): void - { - // Check which of createrepo or createrepo_c is present on the system - if (!file_exists($this->createrepo)) { - throw new Exception('Could not find createrepo on the system'); - } + // Retrieve output from process + $this->taskLogSubStepController->output($process->getOutput(), 'pre'); - // Check if root path exists - if (!is_dir($this->root)) { - throw new Exception("Repository root directory '" . $this->root . "' does not exist"); + if ($process->getExitCode() != 0) { + throw new Exception('Could not generate repository metadata'); } - // If a comps.xml file exists in the root directory, include it in the metadata - if (file_exists($this->root . '/comps.xml')) { - $this->createrepoArgs .= ' --groupfile=' . $this->root . '/comps.xml'; - } + $process->close(); - $this->taskLogSubStepController->new('create-metadata', 'GENERATING REPOSITORY METADATA'); + $this->taskLogSubStepController->completed(); - // Create repository metadata - $myprocess = new Process($this->createrepo . ' ' . $this->createrepoArgs . ' ' . $this->root . '/'); - $myprocess->setBackground(true); - $myprocess->execute(); + $this->taskLogSubStepController->new('add-updateinfo', 'ADDING UPDATEINFO TO METADATA'); - /** - * Retrieve PID of the launched process - * Then write PID to main PID file - */ - $this->taskController->addsubpid($myprocess->getPid()); + // Add additional metadata files to the repository metadata using modifyrepo_c + foreach ($this->additionalMetadataFiles as $file => $value) { + // If the temporary file does not exist, skip the addition of this metadata file + if (!file_exists($this->root . '/' . $value['temp'])) { + continue; + } - // Retrieve output from process - $output = $myprocess->getOutput(); + // Rename the temporary file back to its original name + if (!rename($this->root . '/' . $value['temp'], $this->root . '/' . $file)) { + throw new Exception('Could not rename ' . $this->root . '/' . $value['temp'] . ' to ' . $this->root . '/' . $file); + } - $this->taskLogSubStepController->output($output, 'pre'); + // Add the file to the repository metadata using modifyrepo_c + $process = new Process($this->modifyrepo . ' ' . $this->modifyrepoArgs . ' --mdtype=' . $value['mdtype'] . ' ' . $this->root . '/' . $file . ' ' . $this->root . '/repodata/'); + $process->setBackground(true); + $process->execute(); - if ($myprocess->getExitCode() != 0) { - throw new Exception('Could not generate repository metadata'); - } + // Retrieve PID of the launched process then write PID to main PID file + $this->taskController->addsubpid($process->getPid()); - $myprocess->close(); + // Retrieve output from process + $this->taskLogSubStepController->output($process->getOutput(), 'pre'); - // Add updateinfo to metadata if it exists - $this->taskLogSubStepController->new('add-updateinfo', 'ADDING UPDATEINFO TO METADATA'); - $this->addMetadata('updateinfo.xml', 'updateinfo'); - $this->taskLogSubStepController->completed(); + if ($process->getExitCode() != 0) { + throw new Exception('Could not add ' . $value['mdtype'] . ' to repository metadata'); + } - // Delete temporary metadata files as they are no longer needed - foreach (['comps.xml', 'modules.yaml'] as $file) { - if (file_exists($this->root . '/' . $file)) { - if (!unlink($this->root . '/' . $file)) { - throw new Exception('Could not delete ' . $this->root . '/' . $file); - } + $process->close(); + + // Delete the file as it's now part of the metadata + if (!unlink($this->root . '/' . $file)) { + throw new Exception('Could not delete ' . $this->root . '/' . $file); } } diff --git a/www/controllers/Repo/Mirror/Rpm.php b/www/controllers/Repo/Mirror/Rpm.php index 919e2fad9..1dd32bdfe 100644 --- a/www/controllers/Repo/Mirror/Rpm.php +++ b/www/controllers/Repo/Mirror/Rpm.php @@ -112,8 +112,7 @@ private function downloadModules(string $url) : void } /** - * Download the modules file with a temporary compressed name, then uncompress it to a plain - * modules.yaml file so it can be found and included automatically by createrepo_c. + * Download the modules file with a temporary compressed name, then uncompress it to a plain modules.yaml file */ $modulesFileTargetName = 'modules.yaml' . ($modulesFileExtension != 'yaml' ? '.' . $modulesFileExtension : ''); @@ -127,7 +126,7 @@ private function downloadModules(string $url) : void throw new Exception('' . $modulesFileTargetName . ' checksum does not match provided checksum'); } - // Uncompress the modules file to a plain modules.yaml based on its actual mime type (required for createrepo_c to include it) + // Uncompress the modules file to a plain modules.yaml based on its actual mime type $mime = mime_content_type($this->workingDir . '/' . $modulesFileTargetName); try { @@ -806,10 +805,8 @@ public function mirror() : void * Loop through all the archs selected by the user to build all the possible URLs to explore */ foreach ($this->arch as $arch) { - $url = $this->url; - // Replace $releasever variable in the URL if exists - $url = str_replace('$releasever', $this->releasever, $url); + $url = str_replace('$releasever', $this->releasever, $this->url); // If there is a $basearch variable in the URL, replace it with the current arch if (preg_match('/\$basearch/i', $url)) { @@ -823,8 +820,6 @@ public function mirror() : void // If 'src' exists in the arch array if (in_array('src', $this->arch)) { - $url = $this->url; - // Replace $releasever variable in the URL if exists $url = str_replace('$releasever', $this->releasever, $url); @@ -855,7 +850,7 @@ public function mirror() : void 'sslCertificatePath' => $this->sslCustomCertificate, 'sslPrivateKeyPath' => $this->sslCustomPrivateKey, 'sslCaCertificatePath' => $this->sslCustomCaCertificate, - 'proxy' => PROXY ?? null, + 'proxy' => PROXY ?? null ]); } catch (Exception $e) { // If the URL is not reachable, add it to the errorUrls array diff --git a/www/controllers/Repo/Snapshot/Package.php b/www/controllers/Repo/Snapshot/Package.php index baeff7550..691a2f33f 100644 --- a/www/controllers/Repo/Snapshot/Package.php +++ b/www/controllers/Repo/Snapshot/Package.php @@ -7,6 +7,7 @@ use Controllers\User\Permission\Repo as RepoPermission; use Controllers\Repo\Snapshot\Snapshot; use Controllers\Exception\AppException; +use Controllers\Filesystem\Directory; use Controllers\Filesystem\File; use Controllers\Utils\Validate; use Controllers\Utils\Convert; @@ -312,6 +313,9 @@ public function upload(array $packages, bool $overwrite = false, bool $ignoreIfE $return['Following packages already exist and have been ignored'] = $packageIgnored; } + // Calculate and update snapshot size + $this->repoSnapshotController->updateSize($this->snapId, Directory::getSize($this->snapshotPath)); + return $return; } @@ -366,6 +370,9 @@ public function delete(array $files): array throw new Exception('Nothing to delete'); } + // Calculate and update snapshot size + $this->repoSnapshotController->updateSize($this->snapId, Directory::getSize($this->snapshotPath)); + return $deleted; } diff --git a/www/controllers/Repo/Snapshot/Snapshot.php b/www/controllers/Repo/Snapshot/Snapshot.php index 31b3b15a6..3179e2a83 100644 --- a/www/controllers/Repo/Snapshot/Snapshot.php +++ b/www/controllers/Repo/Snapshot/Snapshot.php @@ -3,6 +3,7 @@ namespace Controllers\Repo\Snapshot; use Controllers\Filesystem\Directory; +use Controllers\Utils\Convert; use Controllers\Repo\Repo; use JsonException; use Exception; @@ -55,6 +56,16 @@ private function getUnused(string $repoId, string $retention) : array return $this->model->getUnused($repoId, $retention); } + /** + * Return the size of a snapshot by ID + * Human-readable format if $human is true, otherwise return size in bytes + * Can return null if the snapshot size is not yet calculated + */ + public function getSize(int $snapId, bool $human = false): int|string|null + { + return $this->model->getSize($snapId, $human); + } + /** * Return the last insert row ID in database */ @@ -110,6 +121,14 @@ public function updateStatus(int $snapId, string $status): void $this->model->updateStatus($snapId, $status); } + /** + * Update snapshot size in the database + */ + public function updateSize(int $snapId, string $size): void + { + $this->model->updateSize($snapId, $size, Convert::sizeToHuman($size)); + } + /** * Update snapshot rebuild status in the database */ @@ -150,7 +169,7 @@ public function clean(int $repoId) : string $removedSnaps = []; $removedSnapsError = []; $removedSnapsFinalArray = []; - $repoController = new \Controllers\Repo\Repo(); + $repoController = new Repo(); $scheduledTaskController = new \Controllers\Task\Scheduled(); // Get repository details diff --git a/www/controllers/Repo/Task/Duplicate.php b/www/controllers/Repo/Task/Duplicate.php index 9d0e7bc07..72722a25d 100644 --- a/www/controllers/Repo/Task/Duplicate.php +++ b/www/controllers/Repo/Task/Duplicate.php @@ -282,6 +282,11 @@ public function execute() */ $targetSnapId = $this->repoSnapshotController->getLastInsertRowID(); + // Calculate and update snapshot size + $this->taskLogSubStepController->new('calculating-size', 'CALCULATING SNAPSHOT SIZE'); + $this->repoSnapshotController->updateSize($targetSnapId, Directory::getSize($targetSnapshotPath)); + $this->taskLogSubStepController->completed(); + /** * Add the new repo environment in database, only if the user has specified an environment */ diff --git a/www/controllers/Repo/Task/Finalize.php b/www/controllers/Repo/Task/Finalize.php index aad2e24f8..e2a9a38da 100644 --- a/www/controllers/Repo/Task/Finalize.php +++ b/www/controllers/Repo/Task/Finalize.php @@ -2,6 +2,7 @@ namespace Controllers\Repo\Task; +use Controllers\Filesystem\Directory; use Controllers\Filesystem\File; use Exception; @@ -187,6 +188,11 @@ protected function finalize() $this->taskLogStepController->completed(); } + // Calculate and update snapshot size, this step can be done before the cleaning step, as temporary files are empty files and will not be counted in the size calculation + $this->taskLogSubStepController->new('calculating-size', 'CALCULATING SNAPSHOT SIZE'); + $this->repoSnapshotController->updateSize($this->repoController->getSnapId(), Directory::getSize($snapshotPath)); + $this->taskLogSubStepController->completed(); + $this->taskLogStepController->new('cleaning', 'CLEANING'); // Clean .completed and .signed files left diff --git a/www/controllers/Service/Unit/Main.php b/www/controllers/Service/Unit/Main.php index 0448bd957..b146d6e94 100644 --- a/www/controllers/Service/Unit/Main.php +++ b/www/controllers/Service/Unit/Main.php @@ -8,6 +8,7 @@ class Main extends \Controllers\Service\Service { private $taskController; private $signalHandler; + private $units; public function __construct(string $unit) { @@ -15,6 +16,10 @@ public function __construct(string $unit) $this->taskController = new \Controllers\Task\Task(); $this->signalHandler = new \Controllers\SignalHandler(); + + // Load service units configuration + include(ROOT . '/config/service-units.php'); + $this->units = $units; } /** @@ -22,14 +27,8 @@ public function __construct(string $unit) */ public function run() : void { - $counter = 0; - $lastScheduledTaskRunning = null; - $lastStatsRunning = null; $startup = true; - // Load service units configuration - include(ROOT . '/config/service-units.php'); - // Main loop, every minute while (true) { $launchedUnits = []; @@ -52,11 +51,11 @@ public function run() : void */ if ($minutesNow == '00' || $startup === true) { // Run hourly unit tasks - foreach ($units as $unitName => $unit) { + foreach ($this->units as $unitName => $unit) { if ($unit['frequency'] == 'every-hour') { // Check if the unit has not already been launched if (!in_array($unitName, $launchedUnits)) { - $this->runUnit($unit['title'], $unitName, isset($unit['force']) ? $unit['force'] : false); + $this->runUnit($unitName, isset($unit['force']) ? $unit['force'] : false); // Add the unit to the list of launched units, to avoid launching it twice in this loop $launchedUnits[] = $unitName; @@ -68,11 +67,11 @@ public function run() : void /** * Run tasks scheduled to run every days at a specific time */ - foreach ($units as $unitName => $unit) { + foreach ($this->units as $unitName => $unit) { if ($unit['frequency'] == 'every-day' && $unit['time'] == $currentTime) { // Check if the unit has not already been launched if (!in_array($unitName, $launchedUnits)) { - $this->runUnit($unit['title'], $unitName, isset($unit['force']) ? $unit['force'] : false); + $this->runUnit($unitName, isset($unit['force']) ? $unit['force'] : false); // Add the unit to the list of launched units, to avoid launching it twice in this loop $launchedUnits[] = $unitName; @@ -83,11 +82,11 @@ public function run() : void /** * Run tasks scheduled to run every weeks on a specific day at a specific time */ - foreach ($units as $unitName => $unit) { + foreach ($this->units as $unitName => $unit) { if ($unit['frequency'] == 'every-week' && isset($unit['day']) && $unit['day'] == strtolower(date('l')) && isset($unit['time']) && $unit['time'] == $currentTime) { // Check if the unit has not already been launched if (!in_array($unitName, $launchedUnits)) { - $this->runUnit($unit['title'], $unitName, isset($unit['force']) ? $unit['force'] : false); + $this->runUnit($unitName, isset($unit['force']) ? $unit['force'] : false); // Add the unit to the list of launched units, to avoid launching it twice in this loop $launchedUnits[] = $unitName; @@ -98,11 +97,11 @@ public function run() : void /** * Finally, run tasks scheduled to run every minutes */ - foreach ($units as $unitName => $unit) { + foreach ($this->units as $unitName => $unit) { if ($unit['frequency'] == 'every-minute' or $unit['frequency'] == 'forever') { // Check if the unit has not already been launched if (!in_array($unitName, $launchedUnits)) { - $this->runUnit($unit['title'], $unitName, isset($unit['force']) ? $unit['force'] : false); + $this->runUnit($unitName, isset($unit['force']) ? $unit['force'] : false); // Add the unit to the list of launched units, to avoid launching it twice in this loop $launchedUnits[] = $unitName; @@ -124,7 +123,7 @@ public function run() : void /** * Run this service with the specified unit name */ - private function runUnit(string $title, string $unit, bool $force = false) : void + public function runUnit(string $unit, bool $force = false) : void { try { /** @@ -147,7 +146,7 @@ private function runUnit(string $title, string $unit, bool $force = false) : voi /** * Else, run the service with the specified unit name */ - parent::logDebug('Running: ' . $title . '...'); + parent::logDebug('Running: ' . $this->units[$unit]['title'] . '...'); $myprocess = new \Controllers\Process("/usr/bin/php " . ROOT . "/tools/service.php '" . $unit . "' >/dev/null 2>/dev/null &"); $myprocess->execute(); @@ -158,7 +157,7 @@ private function runUnit(string $title, string $unit, bool $force = false) : voi throw new Exception($output); } } catch (Exception $e) { - parent::logError('Error while launching ' . $title . ' (service unit ' . $unit . '): ' . $e->getMessage()); + parent::logError('Error while launching ' . $this->units[$unit]['title'] . ' (service unit ' . $unit . '): ' . $e->getMessage()); } } } diff --git a/www/controllers/Service/Unit/Repo/Size.php b/www/controllers/Service/Unit/Repo/Size.php new file mode 100644 index 000000000..353b6a2f7 --- /dev/null +++ b/www/controllers/Service/Unit/Repo/Size.php @@ -0,0 +1,49 @@ +list(); + + foreach ($repos as $repo) { + // Determine the path to the snapshot directory + if ($repo['Package_type'] == 'rpm') { + $snapshotPath = REPOS_DIR . '/rpm/' . $repo['Name'] . '/' . $repo['Releasever'] . '/' . $repo['Date']; + } + if ($repo['Package_type'] == 'deb') { + $snapshotPath = REPOS_DIR . '/deb/' . $repo['Name'] . '/' . $repo['Dist'] . '/' . $repo['Section'] . '/' . $repo['Date']; + } + + if (!is_dir($snapshotPath)) { + parent::logError("Snapshot directory does not exist: $snapshotPath"); + continue; + } + + // Update the size in the database + $repoSnapshotController->updateSize($repo['snapId'], Directory::getSize($snapshotPath)); + } + + parent::log('Repository snapshots size calculation completed.'); + } +} diff --git a/www/controllers/User/Edit.php b/www/controllers/User/Edit.php index ccc10ab03..f31a82291 100644 --- a/www/controllers/User/Edit.php +++ b/www/controllers/User/Edit.php @@ -2,6 +2,7 @@ namespace Controllers\User; use Exception; +use Controllers\App\Session; use Controllers\History\Save as History; use Controllers\Utils\Validate; @@ -53,9 +54,11 @@ public function edit(int $id, string $type, string $firstName = '', string $last /** * Update sessions variables with new values */ - $_SESSION['first_name'] = $firstName; - $_SESSION['last_name'] = $lastName; - $_SESSION['email'] = $email; + Session::set([ + 'first_name' => $firstName, + 'last_name' => $lastName, + 'email' => $email + ]); } /** diff --git a/www/controllers/User/Login.php b/www/controllers/User/Login.php index 784f6d583..57177fce5 100644 --- a/www/controllers/User/Login.php +++ b/www/controllers/User/Login.php @@ -1,6 +1,8 @@ getIdByUsername($username, 'local'); - /** - * If no matching user has been found, throw an exception - */ + // If no matching user has been found, throw an exception if (empty($id)) { throw new Exception('Unknown login'); } - /** - * Checking in database that username/password couple is matching - */ + // Checking in database that username/password couple is matching $this->checkUsernamePwd($id, $password); - /** - * Getting all user informations in datbase - */ + // Getting all user informations in database $informations = $this->get($id); - /** - * Starting session - */ - session_start([ - 'cookie_secure' => (isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] === 'on'), - 'cookie_httponly' => true, + // Saving user informations in session variables + Session::set([ + 'username' => $username, + 'id' => $informations['userId'], + 'role' => $informations['Role_name'], + 'first_name' => $informations['First_name'], + 'last_name' => $informations['Last_name'], + 'email' => $informations['Email'], + 'type' => 'local', ]); - /** - * Saving user informations in session variables - */ - $_SESSION['username'] = $username; - $_SESSION['id'] = $informations['userId']; - $_SESSION['role'] = $informations['Role_name']; - $_SESSION['first_name'] = $informations['First_name']; - $_SESSION['last_name'] = $informations['Last_name']; - $_SESSION['email'] = $informations['Email']; - $_SESSION['type'] = 'local'; - - /** - * Add history - */ + // Add history History::set('Authentication (local account)'); - /** - * If an 'origin' cookie exists then redirect the user to the specified URI - */ + // If an 'origin' cookie exists then redirect the user to the specified URI if (!empty($_COOKIE['origin'])) { if ($_COOKIE['origin'] != '/logout') { header('Location: ' . $_COOKIE['origin']); @@ -78,21 +60,15 @@ public function login(string $username, string $password) : void } } - /** - * Else redirect to default page '/' - */ + // Else redirect to default page '/' header('Location: /'); exit(); } catch (Exception $e) { - /** - * Add history - */ + // Add history History::set('Authentication failed for ' . $username . ' (local account): ' . $e->getMessage(), 'error', $username); - /** - * Throw back an exception with generic message to display on login page - */ + // Throw back an exception with generic message to display on login page throw new Exception('Invalid login and/or password'); } } @@ -110,72 +86,52 @@ public function ssoLogin(): void $email = ''; $role = 'usage'; - /** - * Initialize OpenID Connect client - */ + // Initialize OpenID Connect client $oidc = new OpenIDConnectClient( OIDC_PROVIDER_URL, OIDC_CLIENT_ID, OIDC_CLIENT_SECRET ); - /** - * Disable https upgrade: useful for local/dev environment with no https - */ + // Disable https upgrade: useful for local/dev environment with no https $oidc->setHttpUpgradeInsecureRequests(false); - /** - * Use OIDC_AUTHORIZATION_ENDPOINT as authorization_endpoint if defined - */ + // Use OIDC_AUTHORIZATION_ENDPOINT as authorization_endpoint if defined if (!empty(OIDC_AUTHORIZATION_ENDPOINT)) { $oidc->providerConfigParam(['authorization_endpoint' => OIDC_AUTHORIZATION_ENDPOINT]); } - /** - * Use OIDC_TOKEN_ENDPOINT as token_endpoint if defined - */ + // Use OIDC_TOKEN_ENDPOINT as token_endpoint if defined if (!empty(OIDC_TOKEN_ENDPOINT)) { $oidc->providerConfigParam(['token_endpoint' => OIDC_TOKEN_ENDPOINT]); } - /** - * Use OIDC_USERINFO_ENDPOINT as userinfo_endpoint if defined - */ + // Use OIDC_USERINFO_ENDPOINT as userinfo_endpoint if defined if (!empty(OIDC_USERINFO_ENDPOINT)) { $oidc->providerConfigParam(['userinfo_endpoint' => OIDC_USERINFO_ENDPOINT]); } - /** - * Use OIDC_SCOPES as scopes if defined - */ + // Use OIDC_SCOPES as scopes if defined if (!empty(OIDC_SCOPES)) { // Convert OIDC_SCOPES string to array $scopes = explode(',', OIDC_SCOPES); $oidc->addScope($scopes); } - /** - * Use OIDC_HTTP_PROXY as httpProxy if defined - */ + // Use OIDC_HTTP_PROXY as httpProxy if defined if (!empty(OIDC_HTTP_PROXY)) { $oidc->setHttpProxy(OIDC_HTTP_PROXY); } - /** - * Use OIDC_CERT_PATH as certPath if defined - */ + // Use OIDC_CERT_PATH as certPath if defined if (!empty(OIDC_CERT_PATH)) { $oidc->setCertPath(OIDC_CERT_PATH); } - /** - * Try to authenticate user - */ + // Try to authenticate user $oidc->authenticate(); - /** - * Get user informations - */ + // Get user informations $roles = $oidc->getVerifiedClaims(OIDC_GROUPS); $username = $oidc->getVerifiedClaims(OIDC_USERNAME); $firstName = $oidc->requestUserInfo(OIDC_FIRST_NAME); @@ -186,9 +142,7 @@ public function ssoLogin(): void throw new Exception('No username found in SSO response'); } - /** - * Define user role based on OIDC_GROUPS - */ + // Define user role based on OIDC_GROUPS if (is_array($roles)) { if (!empty(OIDC_GROUP_ADMINISTRATOR) && in_array(OIDC_GROUP_ADMINISTRATOR, $roles)) { $role = 'administrator'; @@ -198,34 +152,28 @@ public function ssoLogin(): void // } } - /** - * Saving user informations in session variable - */ - $_SESSION['username'] = $username; - $_SESSION['role'] = $role; - $_SESSION['first_name'] = $firstName; - $_SESSION['last_name'] = $lastName; - $_SESSION['email'] = $email; - $_SESSION['type'] = 'sso'; + // Saving user informations in session variable + Session::set([ + 'username' => $username, + 'role' => $role, + 'first_name' => $firstName, + 'last_name' => $lastName, + 'email' => $email, + 'type' => 'sso', + ]); - /** - * Create user in database - */ + // Create user in database $userCreateController->createSSO($username, $firstName, $lastName, $email, $role); - /** - * Also save user Id in session variable now that the user exists in database - */ - $_SESSION['id'] = $this->getIdByUsername($username, 'sso'); + // Also save user Id in session variable now that the user exists in database + Session::set([ + 'id' => $this->getIdByUsername($username, 'sso'), + ]); - /** - * Add history - */ + // Add history History::set('Authentication (SSO account)'); - /** - * If an 'origin' cookie exists then redirect the user to the specified URI - */ + // If an 'origin' cookie exists then redirect the user to the specified URI if (!empty($_COOKIE['origin'])) { if ($_COOKIE['origin'] != '/logout') { header('Location: ' . $_COOKIE['origin']); @@ -233,9 +181,7 @@ public function ssoLogin(): void } } - /** - * Else redirect to default page '/' - */ + // Else redirect to default page '/' header('Location: /'); exit(); @@ -266,9 +212,7 @@ public function ssoLogin(): void // Throw exception to display error message on login page throw new Exception($error); - /** - * If debug mode is disabled, just display the error message - */ + // If debug mode is disabled, just display the error message } else { // Throw exception to display error message on login page throw new Exception('Could not connect through SSO: ' . $e->getMessage()); diff --git a/www/controllers/ajax/repo/get.php b/www/controllers/ajax/repo/get.php index 45af1220d..b815d770d 100644 --- a/www/controllers/ajax/repo/get.php +++ b/www/controllers/ajax/repo/get.php @@ -1,27 +1,6 @@ getMessage()); - } - - response(HTTP_OK, $size); -} - /** * Get latest tasks status */ diff --git a/www/models/Connection.php b/www/models/Connection.php index adb848113..d2c513b40 100644 --- a/www/models/Connection.php +++ b/www/models/Connection.php @@ -280,13 +280,16 @@ private function generateMainTables(): void Advanced_params TEXT, Type CHAR(6) NOT NULL, Reconstruct CHAR(8), /* needed, running, failed */ + Size INTEGER, /* size in bytes */ + Size_human VARCHAR(255), /* human readable size */ Status CHAR(8) NOT NULL, Id_repo INTEGER NOT NULL)"); /** * Create indexes */ - $this->exec("CREATE INDEX IF NOT EXISTS idx_repos_snap ON repos_snap (Date, Time, Signed, Arch, Type, Reconstruct, Status, Id_repo)"); + // TODO: uncomment in release > 5.14.0 + // $this->exec("CREATE INDEX IF NOT EXISTS idx_repos_snap ON repos_snap (Date, Time, Signed, Arch, Type, Reconstruct, Size, Size_human, Status, Id_repo)"); $this->exec("CREATE INDEX IF NOT EXISTS idx_repos_snap_status_id_repo ON repos_snap (Status, Id_repo)"); $this->exec("CREATE INDEX IF NOT EXISTS idx_repos_snap_id_repo ON repos_snap (Id_repo)"); diff --git a/www/models/Repo/Listing.php b/www/models/Repo/Listing.php index 83d7f2e28..6fc56f6f6 100644 --- a/www/models/Repo/Listing.php +++ b/www/models/Repo/Listing.php @@ -36,6 +36,9 @@ public function list() : array repos_snap.Signed, repos_snap.Arch, repos_snap.Type, + repos_snap.Reconstruct, + repos_snap.Size, + repos_snap.Size_human, repos_env.Description FROM repos LEFT JOIN repos_snap @@ -82,6 +85,8 @@ public function listByGroup(string $groupName) : array repos_snap.Arch, repos_snap.Type, repos_snap.Reconstruct, + repos_snap.Size, + repos_snap.Size_human, repos_snap.Status, repos_env.Description FROM repos @@ -109,6 +114,8 @@ public function listByGroup(string $groupName) : array repos_snap.Arch, repos_snap.Type, repos_snap.Reconstruct, + repos_snap.Size, + repos_snap.Size_human, repos_snap.Status, repos_env.Description FROM repos diff --git a/www/models/Repo/Repo.php b/www/models/Repo/Repo.php index 5cf8a11a8..f1f005500 100644 --- a/www/models/Repo/Repo.php +++ b/www/models/Repo/Repo.php @@ -60,6 +60,8 @@ public function getAllById(string|null $repoId, string|null $snapId, string|null repos_snap.Advanced_params, repos_snap.Type, repos_snap.Reconstruct, + repos_snap.Size, + repos_snap.Size_human, repos_snap.Status, repos_snap.Id_repo, repos_env.Id AS envId, @@ -94,6 +96,8 @@ public function getAllById(string|null $repoId, string|null $snapId, string|null repos_snap.Advanced_params, repos_snap.Type, repos_snap.Reconstruct, + repos_snap.Size, + repos_snap.Size_human, repos_snap.Status, repos_snap.Id_repo FROM repos @@ -132,6 +136,8 @@ public function getAllById(string|null $repoId, string|null $snapId, string|null repos_snap.Advanced_params, repos_snap.Type, repos_snap.Reconstruct, + repos_snap.Size, + repos_snap.Size_human, repos_snap.Status, repos_snap.Id_repo FROM repos @@ -156,6 +162,8 @@ public function getAllById(string|null $repoId, string|null $snapId, string|null repos_snap.Advanced_params, repos_snap.Type, repos_snap.Reconstruct, + repos_snap.Size, + repos_snap.Size_human, repos_snap.Status, repos_snap.Id_repo, repos_env.Id AS envId, diff --git a/www/models/Repo/Snapshot/Snapshot.php b/www/models/Repo/Snapshot/Snapshot.php index 3ede45612..a49f87c61 100644 --- a/www/models/Repo/Snapshot/Snapshot.php +++ b/www/models/Repo/Snapshot/Snapshot.php @@ -88,6 +88,33 @@ public function getUnused(string $repoId, string $retention) : array return $data; } + /** + * Return the size of a snapshot by ID + */ + public function getSize(int $snapId, bool $human): int|string|null + { + $data = 0; + + try { + if ($human) { + $stmt = $this->db->prepare("SELECT Size_human as Size FROM repos_snap WHERE Id = :snapId"); + } else { + $stmt = $this->db->prepare("SELECT Size FROM repos_snap WHERE Id = :snapId"); + } + + $stmt->bindValue(':snapId', $snapId); + $result = $stmt->execute(); + } catch (Exception $e) { + DbLog::error($e); + } + + while ($row = $result->fetchArray(SQLITE3_ASSOC)) { + $data = $row['Size']; + } + + return $data; + } + /** * Add a snapshot in database */ @@ -157,7 +184,7 @@ public function updateGpgSignature(int $snapId, string $gpgSignature): void /** * Update snapshot status in the database */ - public function updateStatus(string $snapId, string $status): void + public function updateStatus(int $snapId, string $status): void { try { $stmt = $this->db->prepare("UPDATE repos_snap SET Status = :status WHERE Id = :snapId"); @@ -169,6 +196,19 @@ public function updateStatus(string $snapId, string $status): void } } + public function updateSize(int $snapId, string $size, string $sizeHuman): void + { + try { + $stmt = $this->db->prepare("UPDATE repos_snap SET Size = :size, Size_human = :sizeHuman WHERE Id = :snapId"); + $stmt->bindValue(':size', $size); + $stmt->bindValue(':sizeHuman', $sizeHuman); + $stmt->bindValue(':snapId', $snapId); + $stmt->execute(); + } catch (Exception $e) { + DbLog::error($e); + } + } + /** * Update snapshot rebuild status in the database */ diff --git a/www/public/resources/js/classes/Repo.js b/www/public/resources/js/classes/Repo.js index 0198e1bc2..8e9163358 100644 --- a/www/public/resources/js/classes/Repo.js +++ b/www/public/resources/js/classes/Repo.js @@ -87,38 +87,6 @@ class Repo { mylayout.hideLoading(); } - /** - * Get repositories size - */ - getSize() - { - // Loop through all repos and get their size - $('#repos-list-container').find('.item-size').each(function () { - var repoId = $(this).attr('repo-id'); - var snapId = $(this).attr('snap-id'); - var path = $(this).attr('repo-relative-path'); - - ajaxRequest( - // Controller: - 'repo/get', - // Action: - 'size', - // Data: - { - path: path - }, - // Print success alert: - false, - // Print error alert: - false - ).then(function () { - $("#repos-list-container").find('.item-size[repo-id="' + repoId + '"][snap-id="' + snapId + '"]').html(jsonValue.message); - }).catch(function () { - $("#repos-list-container").find('.item-size[repo-id="' + repoId + '"][snap-id="' + snapId + '"]').replaceWith(''); - }); - }); - } - /** * Get latest task status for all repos */ diff --git a/www/public/resources/js/events/repo/install.js b/www/public/resources/js/events/repo/install.js index c3636753c..2d54d0562 100644 --- a/www/public/resources/js/events/repo/install.js +++ b/www/public/resources/js/events/repo/install.js @@ -32,7 +32,7 @@ $(document).on('change','#repo-install-select-env',function (e) { // deb822 syntax if (packageType == 'deb-alt') { - html = 'cat << EOF > /etc/apt/sources.list.d/' + prefix + name + '-' + dist + '-' + component + '.list\n'; + html = 'cat << EOF > /etc/apt/sources.list.d/' + prefix + name + '-' + dist + '-' + component + '.sources\n'; html += 'Types: deb\n'; html += 'URIs: ' + url + '/deb/' + name + '/' + dist + '/' + component + '/' + env + '\n'; html += 'Suites: ' + dist + '\n'; diff --git a/www/public/resources/js/functions/browse.js b/www/public/resources/js/functions/browse.js index 23dbc614d..97418185f 100644 --- a/www/public/resources/js/functions/browse.js +++ b/www/public/resources/js/functions/browse.js @@ -45,6 +45,8 @@ function deletePackages(snapId) { var packages = []; + myalert.print('Deleting packages...'); + // Get the path of the selected packages $('body').find('input[name=packageName\\[\\]]:checked').each(function () { packages.push($(this).attr('path')); diff --git a/www/templates/source-repositories/rpm/centos.yml b/www/templates/source-repositories/rpm/centos.yml index 65b8a0e97..1100057ef 100644 --- a/www/templates/source-repositories/rpm/centos.yml +++ b/www/templates/source-repositories/rpm/centos.yml @@ -5,7 +5,7 @@ repositories: # CentOS BaseOS - name: centos-baseos type: rpm - description: CentOS 9 Stream BaseOS + description: CentOS Stream BaseOS url: https://mirror.stream.centos.org/$releasever-stream/BaseOS/$basearch/os releasever: - name: 10 @@ -21,7 +21,7 @@ repositories: # CentOS AppStream - name: centos-appstream type: rpm - description: CentOS 9 Stream AppStream + description: CentOS Stream AppStream url: https://mirror.stream.centos.org/$releasever-stream/AppStream/$basearch/os releasever: - name: 10 @@ -37,15 +37,15 @@ repositories: # CentOS Extras Common - name: centos-extras-common type: rpm - description: CentOS 9 Stream Extras Common + description: CentOS Stream Extras Common url: https://mirror.stream.centos.org/SIGs/$releasever-stream/extras/$basearch/extras-common releasever: - name: 10 description: CentOS 10 Stream gpgkeys: - - fingerprint: 99DB70FAE1D7CE227FB6488205B555B38483C65D + - fingerprint: 363FC0972F64B699AED3968E1FF6A2171D997668 - name: 9 description: CentOS 9 Stream gpgkeys: - - fingerprint: 99DB70FAE1D7CE227FB6488205B555B38483C65D + - fingerprint: 363FC0972F64B699AED3968E1FF6A2171D997668 diff --git a/www/tools/service.php b/www/tools/service.php index 51403c2f1..03c41eada 100644 --- a/www/tools/service.php +++ b/www/tools/service.php @@ -1,15 +1,19 @@ getMessage() . PHP_EOL . $e->getTraceAsString() . PHP_EOL); $logController->log('error', 'Background service', 'General error: ' . $e->getMessage(), $e->getTraceAsString()); diff --git a/www/update/database/5.14.0.php b/www/update/database/5.14.0.php new file mode 100644 index 000000000..a027fbb32 --- /dev/null +++ b/www/update/database/5.14.0.php @@ -0,0 +1,36 @@ +db->exec("DROP INDEX IF EXISTS idx_repos_snap"); +} catch (Exception $e) { + throw new Exception('could not delete old indexes from main database: ' . $e->getMessage()); +} + +// Add 'Size' column to repos_snap table +if (!$this->db->columnExist('repos_snap', 'Size')) { + $this->db->exec("ALTER TABLE repos_snap ADD COLUMN Size INTEGER"); +} + +// Add 'Size_human' column to repos_snap table +if (!$this->db->columnExist('repos_snap', 'Size_human')) { + $this->db->exec("ALTER TABLE repos_snap ADD COLUMN Size_human VARCHAR(255)"); +} + +// Create new indexes +try { + $this->db->exec("CREATE INDEX IF NOT EXISTS idx_repos_snap ON repos_snap (Date, Time, Signed, Arch, Type, Reconstruct, Size, Size_human, Status, Id_repo)"); +} catch (Exception $e) { + throw new Exception('could not create new indexes for main database: ' . $e->getMessage()); +} + +try { + $serviceUnit = new ServiceUnit('snapshots-size-calculation'); + $serviceUnit->runUnit('snapshots-size-calculation'); +} catch (Exception $e) { + throw new Exception('Error while launching the calculation of snapshots size: ' . $e->getMessage()); +} diff --git a/www/version b/www/version index 4b6549774..5d4f567eb 100644 --- a/www/version +++ b/www/version @@ -1 +1 @@ -5.13.4 \ No newline at end of file +5.14.0 \ No newline at end of file diff --git a/www/views/includes/repos-list.inc.php b/www/views/includes/repos-list.inc.php index 03267d3c1..79a6cb611 100644 --- a/www/views/includes/repos-list.inc.php +++ b/www/views/includes/repos-list.inc.php @@ -96,6 +96,8 @@ $source = $repo['Source']; $rebuild = $repo['Reconstruct']; $status = $repo['Status']; + $size = $repo['Size'] ?? '?'; + $sizeHuman = $repo['Size_human'] ?? '?'; $packageType = $repo['Package_type']; $date = $repo['Date']; $dateFormatted = DateTime::createFromFormat('Y-m-d', $repo['Date'])->format('d-m-Y'); @@ -285,16 +287,6 @@ endif; endif ?> - -
- Calc. +
@@ -425,7 +417,6 @@ \ No newline at end of file