From 20cb943150dfeaa26afdc0ebb643405218384c0f Mon Sep 17 00:00:00 2001 From: bhoffman20 Date: Sun, 4 Oct 2026 18:40:10 -0500 Subject: [PATCH] Load Bookshelf books per author as rows scroll into view The Bookshelf fetched every book of the selected media type up front, which runs large libraries out of memory. Rows now come from per-author book counts, and each row loads its books in batches when it renders. --- frontend/src/Bookshelf/BookshelfConnector.js | 70 +++---- frontend/src/Bookshelf/BookshelfRow.css | 4 + frontend/src/Bookshelf/BookshelfRow.css.d.ts | 1 + frontend/src/Bookshelf/BookshelfRow.js | 11 ++ .../src/Bookshelf/BookshelfRowConnector.js | 41 +++- .../src/Store/Actions/bookshelfActions.js | 186 +++++++++++++++++- src/Chaptarr.Api.V1/Books/BookController.cs | 54 +++++ .../Books/BookshelfAuthorCountResource.cs | 8 + .../BookControllerSiblingDeleteInfoFixture.cs | 1 + .../Wanted/WantedMonitoringFilterFixture.cs | 1 + .../AuthorStatisticsImportDeferralFixture.cs | 1 + .../AuthorStatisticsServiceFixture.cs | 44 ++++- .../Books/BookRepositoryIndexFilterFixture.cs | 33 ++++ ...BookServiceGetBooksLoadsEditionsFixture.cs | 98 +++++++++ .../AuthorStats/AuthorStatisticsRepository.cs | 14 ++ .../AuthorStats/AuthorStatisticsService.cs | 51 ++++- .../Books/Repositories/BookRepository.cs | 28 +++ .../Books/Services/BookService.cs | 72 ++++++- 18 files changed, 662 insertions(+), 56 deletions(-) create mode 100644 src/Chaptarr.Api.V1/Books/BookshelfAuthorCountResource.cs diff --git a/frontend/src/Bookshelf/BookshelfConnector.js b/frontend/src/Bookshelf/BookshelfConnector.js index f26c4c3f..2b478078 100644 --- a/frontend/src/Bookshelf/BookshelfConnector.js +++ b/frontend/src/Bookshelf/BookshelfConnector.js @@ -3,51 +3,37 @@ import React, { Component } from 'react'; import { connect } from 'react-redux'; import { createSelector } from 'reselect'; import { setSelectedMediaType } from 'Store/Actions/appActions'; -import { clearBooks, fetchBooks } from 'Store/Actions/bookActions'; -import { saveBookshelf, setBookshelfFilter, setBookshelfSort } from 'Store/Actions/bookshelfActions'; +import { clearBookshelf, fetchBookshelfAuthors, saveBookshelf, setBookshelfFilter, setBookshelfSort } from 'Store/Actions/bookshelfActions'; import createAuthorClientSideCollectionItemsSelector from 'Store/Selectors/createAuthorClientSideCollectionItemsSelector'; import createDimensionsSelector from 'Store/Selectors/createDimensionsSelector'; import Bookshelf from './Bookshelf'; -function createBookFetchStateSelector() { - return createSelector( - (state) => state.books.items, - (state) => state.books.isFetching, - (state) => state.books.isPopulated, - (state) => state.app.selectedMediaType || 'audiobook', - (items, isFetching, isPopulated, selectedMediaType) => { - const scopedItems = items.filter((book) => book.mediaType === selectedMediaType); - return { - isFetching, - isPopulated, - items: scopedItems, - selectedMediaType - }; - } - ); -} - function createMapStateToProps() { return createSelector( - createBookFetchStateSelector(), + (state) => state.bookshelf, + (state) => state.app.selectedMediaType || 'audiobook', createAuthorClientSideCollectionItemsSelector('bookshelf'), createDimensionsSelector(), - (books, author, dimensionsState) => { - const isPopulated = books.isPopulated && author.isPopulated; - const isFetching = author.isFetching || books.isFetching; - const authorIds = new Set(books.items.map((book) => book.authorId)); - const items = author.items.filter((item) => authorIds.has(item.id)); - const visibleAuthorIds = new Set(items.map((item) => item.id)); - const bookCount = books.items.filter((book) => visibleAuthorIds.has(book.authorId)).length; + (bookshelf, selectedMediaType, author, dimensionsState) => { + const { authorBookCounts } = bookshelf; + const isPopulated = bookshelf.isPopulated && author.isPopulated; + const isFetching = author.isFetching || bookshelf.isFetching; + const items = author.items.filter((item) => authorBookCounts.hasOwnProperty(item.id)); + + let bookCount = 0; + items.forEach((item) => { + bookCount += authorBookCounts[item.id]; + }); return { ...author, items, - totalItems: authorIds.size, + totalItems: Object.keys(authorBookCounts).length, isPopulated, isFetching, + error: author.error || bookshelf.error, bookCount, - selectedMediaType: books.selectedMediaType, + selectedMediaType, isSmallScreen: dimensionsState.isSmallScreen }; } @@ -58,8 +44,8 @@ const mapDispatchToProps = { setBookshelfSort, setBookshelfFilter, setSelectedMediaType, - clearBooks, - fetchBooks, + clearBookshelf, + fetchBookshelfAuthors, saveBookshelf }; @@ -69,24 +55,23 @@ class BookshelfConnector extends Component { // Lifecycle componentDidMount() { - this.fetchBooksForSelectedMediaType(); + // Other pages can replace the books store, so rows reload their books on every visit. + this.props.clearBookshelf(); + this.fetchAuthorBookCounts(); } componentDidUpdate(prevProps) { if (prevProps.selectedMediaType !== this.props.selectedMediaType) { - this.fetchBooksForSelectedMediaType(); + this.fetchAuthorBookCounts(); } } componentWillUnmount() { - if (this.abortBooksFetch) { - this.abortBooksFetch(); - this.abortBooksFetch = null; - } + this.props.clearBookshelf(); } - fetchBooksForSelectedMediaType() { - this.abortBooksFetch = this.props.fetchBooks({ + fetchAuthorBookCounts() { + this.props.fetchBookshelfAuthors({ mediaType: this.props.selectedMediaType }); } @@ -104,7 +89,6 @@ class BookshelfConnector extends Component { onMediaTypeChange = (mediaType) => { this.props.setSelectedMediaType({ mediaType }); - this.props.clearBooks(); }; onUpdateSelectedPress = (payload) => { @@ -132,8 +116,8 @@ BookshelfConnector.propTypes = { setBookshelfSort: PropTypes.func.isRequired, setBookshelfFilter: PropTypes.func.isRequired, setSelectedMediaType: PropTypes.func.isRequired, - clearBooks: PropTypes.func.isRequired, - fetchBooks: PropTypes.func.isRequired, + clearBookshelf: PropTypes.func.isRequired, + fetchBookshelfAuthors: PropTypes.func.isRequired, saveBookshelf: PropTypes.func.isRequired }; diff --git a/frontend/src/Bookshelf/BookshelfRow.css b/frontend/src/Bookshelf/BookshelfRow.css index 8a0cd277..ef9a44ed 100644 --- a/frontend/src/Bookshelf/BookshelfRow.css +++ b/frontend/src/Bookshelf/BookshelfRow.css @@ -42,3 +42,7 @@ flex-wrap: wrap; min-width: 400px; } + +.booksLoading { + margin: 0; +} diff --git a/frontend/src/Bookshelf/BookshelfRow.css.d.ts b/frontend/src/Bookshelf/BookshelfRow.css.d.ts index 74f688d4..89b0f667 100644 --- a/frontend/src/Bookshelf/BookshelfRow.css.d.ts +++ b/frontend/src/Bookshelf/BookshelfRow.css.d.ts @@ -2,6 +2,7 @@ // Please do not change this file! interface CssExports { 'books': string; + 'booksLoading': string; 'cell': string; 'selectCell': string; 'status': string; diff --git a/frontend/src/Bookshelf/BookshelfRow.js b/frontend/src/Bookshelf/BookshelfRow.js index 7dbcf225..3c0d1587 100644 --- a/frontend/src/Bookshelf/BookshelfRow.js +++ b/frontend/src/Bookshelf/BookshelfRow.js @@ -3,6 +3,7 @@ import React, { Component } from 'react'; import AuthorNameLink from 'Author/AuthorNameLink'; import { getAuthorStatusDetails } from 'Author/AuthorStatus'; import Icon from 'Components/Icon'; +import LoadingIndicator from 'Components/Loading/LoadingIndicator'; import VirtualTableRowCell from 'Components/Table/Cells/VirtualTableRowCell'; import VirtualTableSelectCell from 'Components/Table/Cells/VirtualTableSelectCell'; import BookshelfBook from './BookshelfBook'; @@ -20,6 +21,7 @@ class BookshelfRow extends Component { titleSlug, authorName, books, + isBooksLoaded, isSelected, onSelectedChange, onBookMonitoredPress @@ -54,6 +56,14 @@ class BookshelfRow extends Component { + { + !isBooksLoaded && !books.length && + + } + { books.map((book) => { return ( @@ -77,6 +87,7 @@ BookshelfRow.propTypes = { titleSlug: PropTypes.string.isRequired, authorName: PropTypes.string.isRequired, books: PropTypes.arrayOf(PropTypes.object).isRequired, + isBooksLoaded: PropTypes.bool.isRequired, isSelected: PropTypes.bool, onSelectedChange: PropTypes.func.isRequired, onBookMonitoredPress: PropTypes.func.isRequired diff --git a/frontend/src/Bookshelf/BookshelfRowConnector.js b/frontend/src/Bookshelf/BookshelfRowConnector.js index 8f6f7ee2..dc92a412 100644 --- a/frontend/src/Bookshelf/BookshelfRowConnector.js +++ b/frontend/src/Bookshelf/BookshelfRowConnector.js @@ -4,6 +4,7 @@ import React, { Component } from 'react'; import { connect } from 'react-redux'; import { createSelector } from 'reselect'; import { toggleBooksMonitored } from 'Store/Actions/bookActions'; +import { fetchBookshelfBooks, getBookshelfBookKey } from 'Store/Actions/bookshelfActions'; import createAuthorSelector from 'Store/Selectors/createAuthorSelector'; import BookshelfRow from './BookshelfRow'; @@ -23,7 +24,8 @@ function createMapStateToProps() { createAuthorSelector(), getBookMap, (state, props) => props.selectedMediaType, - (author, bookMap, selectedMediaType) => { + (state, props) => state.bookshelf.loadedBookKeys[getBookshelfBookKey(props.selectedMediaType, props.authorId)] === true, + (author, bookMap, selectedMediaType, isBooksLoaded) => { const booksInAuthor = bookMap.hasOwnProperty(author.id) ? bookMap[author.id] : []; const sortedBooks = _.orderBy( booksInAuthor.filter((book) => book.mediaType === selectedMediaType), @@ -36,18 +38,47 @@ function createMapStateToProps() { authorId: author.id, authorName: author.authorName, status: author.status, - books: sortedBooks + books: sortedBooks, + isBooksLoaded }; } ); } const mapDispatchToProps = { - toggleBooksMonitored + toggleBooksMonitored, + fetchBookshelfBooks }; class BookshelfRowConnector extends Component { + // + // Lifecycle + + componentDidMount() { + this.fetchBooksIfNeeded(); + } + + componentDidUpdate() { + this.fetchBooksIfNeeded(); + } + + fetchBooksIfNeeded() { + const { + authorId, + selectedMediaType, + isBooksLoaded + } = this.props; + + // Repeat calls are ignored until the author's books arrive. + if (!isBooksLoaded) { + this.props.fetchBookshelfBooks({ + authorId, + mediaType: selectedMediaType + }); + } + } + // // Listeners @@ -75,7 +106,9 @@ class BookshelfRowConnector extends Component { BookshelfRowConnector.propTypes = { authorId: PropTypes.number.isRequired, selectedMediaType: PropTypes.oneOf(['audiobook', 'ebook']).isRequired, - toggleBooksMonitored: PropTypes.func.isRequired + isBooksLoaded: PropTypes.bool.isRequired, + toggleBooksMonitored: PropTypes.func.isRequired, + fetchBookshelfBooks: PropTypes.func.isRequired }; export default connect(createMapStateToProps, mapDispatchToProps)(BookshelfRowConnector); diff --git a/frontend/src/Store/Actions/bookshelfActions.js b/frontend/src/Store/Actions/bookshelfActions.js index 6f695bcc..7bcfd0a1 100644 --- a/frontend/src/Store/Actions/bookshelfActions.js +++ b/frontend/src/Store/Actions/bookshelfActions.js @@ -1,4 +1,5 @@ import { createAction } from 'redux-actions'; +import { batchActions } from 'redux-batched-actions'; import { filterBuilderTypes, filterBuilderValueTypes, filterTypePredicates, sortDirections } from 'Helpers/Props'; import { createThunk, handleThunks } from 'Store/thunks'; import { isAuthorMonitoredForSelection } from 'Utilities/Author/getAuthorMediaTypeMonitoringStatus'; @@ -6,7 +7,7 @@ import createAjaxRequest from 'Utilities/createAjaxRequest'; import translate from 'Utilities/String/translate'; import { SET_SELECTED_MEDIA_TYPE } from './appActions'; import { filterPredicates, filters } from './authorActions'; -import { set } from './baseActions'; +import { set, update } from './baseActions'; import createHandleActions from './Creators/createHandleActions'; import createSetClientSideCollectionFilterReducer from './Creators/Reducers/createSetClientSideCollectionFilterReducer'; import createSetClientSideCollectionSortReducer from './Creators/Reducers/createSetClientSideCollectionSortReducer'; @@ -16,12 +17,29 @@ import createSetClientSideCollectionSortReducer from './Creators/Reducers/create export const section = 'bookshelf'; +// Shelf rows load their books in batches as they scroll into view, so a large +// library never has to send every book at once. +const BOOK_BATCH_DELAY_MS = 50; +const MAX_AUTHORS_PER_BOOK_REQUEST = 50; + +let pendingAuthorIds = { audiobook: new Set(), ebook: new Set() }; +let requestedBookKeys = new Set(); +let pendingFlushTimeout = null; +let bookRequestGeneration = 0; +let abortAuthorCountsRequest = null; + // // State export const defaultState = { isSaving: false, saveError: null, + isFetching: false, + isPopulated: false, + error: null, + authorBookCounts: {}, + authorBookCountsMediaType: null, + loadedBookKeys: {}, sortKey: 'sortName', sortDirection: sortDirections.ASCENDING, secondarySortKey: 'sortName', @@ -92,6 +110,9 @@ export const persistState = [ export const SET_BOOKSHELF_SORT = 'bookshelf/setBookshelfSort'; export const SET_BOOKSHELF_FILTER = 'bookshelf/setBookshelfFilter'; export const SAVE_BOOKSHELF = 'bookshelf/saveBookshelf'; +export const FETCH_BOOKSHELF_AUTHORS = 'bookshelf/fetchBookshelfAuthors'; +export const FETCH_BOOKSHELF_BOOKS = 'bookshelf/fetchBookshelfBooks'; +export const CLEAR_BOOKSHELF = 'bookshelf/clearBookshelf'; // // Action Creators @@ -99,11 +120,174 @@ export const SAVE_BOOKSHELF = 'bookshelf/saveBookshelf'; export const setBookshelfSort = createAction(SET_BOOKSHELF_SORT); export const setBookshelfFilter = createAction(SET_BOOKSHELF_FILTER); export const saveBookshelf = createThunk(SAVE_BOOKSHELF); +export const fetchBookshelfAuthors = createThunk(FETCH_BOOKSHELF_AUTHORS); +export const fetchBookshelfBooks = createThunk(FETCH_BOOKSHELF_BOOKS); +export const clearBookshelf = createThunk(CLEAR_BOOKSHELF); + +// +// Helpers + +export function getBookshelfBookKey(mediaType, authorId) { + return `${mediaType}-${authorId}`; +} + +function requestBookshelfBooks(getState, dispatch, authorIds, mediaType, generation) { + const { request } = createAjaxRequest({ + url: '/book/shelf', + data: { + authorIds, + mediaType + }, + traditional: true + }); + + request.done((data) => { + if (generation !== bookRequestGeneration) { + return; + } + + // Replace these authors' books for this media type, and keep everything else. + const authorIdSet = new Set(authorIds); + const otherBooks = getState().books.items.filter((book) => { + return !(authorIdSet.has(book.authorId) && book.mediaType === mediaType); + }); + + const loadedBookKeys = { ...getState().bookshelf.loadedBookKeys }; + authorIds.forEach((authorId) => { + loadedBookKeys[getBookshelfBookKey(mediaType, authorId)] = true; + }); + + dispatch(batchActions([ + update({ section: 'books', data: otherBooks.concat(data) }), + set({ section, loadedBookKeys }) + ])); + }); + + request.fail(() => { + if (generation !== bookRequestGeneration) { + return; + } + + // Let the rows ask again the next time they render. + authorIds.forEach((authorId) => { + requestedBookKeys.delete(getBookshelfBookKey(mediaType, authorId)); + }); + }); +} + +function flushBookshelfBookRequests(getState, dispatch) { + pendingFlushTimeout = null; + + Object.keys(pendingAuthorIds).forEach((mediaType) => { + const authorIds = Array.from(pendingAuthorIds[mediaType]); + pendingAuthorIds[mediaType] = new Set(); + + for (let i = 0; i < authorIds.length; i += MAX_AUTHORS_PER_BOOK_REQUEST) { + const batch = authorIds.slice(i, i + MAX_AUTHORS_PER_BOOK_REQUEST); + requestBookshelfBooks(getState, dispatch, batch, mediaType, bookRequestGeneration); + } + }); +} // // Action Handlers export const actionHandlers = handleThunks({ + [FETCH_BOOKSHELF_AUTHORS]: function(getState, payload, dispatch) { + const { mediaType } = payload; + + if (abortAuthorCountsRequest) { + abortAuthorCountsRequest(); + abortAuthorCountsRequest = null; + } + + // Counts for the other media type would list the wrong authors while this loads. + const isSameMediaType = getState().bookshelf.authorBookCountsMediaType === mediaType; + + dispatch(set({ + section, + isFetching: true, + isPopulated: isSameMediaType && getState().bookshelf.isPopulated + })); + + const { request, abortRequest } = createAjaxRequest({ + url: '/book/shelf/authors', + data: { mediaType } + }); + + abortAuthorCountsRequest = abortRequest; + + request.done((data) => { + abortAuthorCountsRequest = null; + + const authorBookCounts = {}; + data.forEach((count) => { + authorBookCounts[count.authorId] = count.bookCount; + }); + + dispatch(set({ + section, + isFetching: false, + isPopulated: true, + error: null, + authorBookCounts, + authorBookCountsMediaType: mediaType + })); + }); + + request.fail((xhr) => { + abortAuthorCountsRequest = null; + + dispatch(set({ + section, + isFetching: false, + isPopulated: false, + error: xhr.aborted ? null : xhr + })); + }); + }, + + [FETCH_BOOKSHELF_BOOKS]: function(getState, payload, dispatch) { + const { authorId, mediaType } = payload; + const key = getBookshelfBookKey(mediaType, authorId); + + if (requestedBookKeys.has(key)) { + return; + } + + requestedBookKeys.add(key); + pendingAuthorIds[mediaType].add(authorId); + + if (!pendingFlushTimeout) { + pendingFlushTimeout = setTimeout(() => { + flushBookshelfBookRequests(getState, dispatch); + }, BOOK_BATCH_DELAY_MS); + } + }, + + [CLEAR_BOOKSHELF]: function(getState, payload, dispatch) { + // Responses from before the clear are dropped by the generation check. + bookRequestGeneration++; + clearTimeout(pendingFlushTimeout); + pendingFlushTimeout = null; + pendingAuthorIds = { audiobook: new Set(), ebook: new Set() }; + requestedBookKeys = new Set(); + + if (abortAuthorCountsRequest) { + abortAuthorCountsRequest(); + abortAuthorCountsRequest = null; + } + + dispatch(set({ + section, + isFetching: false, + isPopulated: false, + error: null, + authorBookCounts: {}, + authorBookCountsMediaType: null, + loadedBookKeys: {} + })); + }, [SAVE_BOOKSHELF]: function(getState, payload, dispatch) { const { diff --git a/src/Chaptarr.Api.V1/Books/BookController.cs b/src/Chaptarr.Api.V1/Books/BookController.cs index d7ecddcb..42dfb2b0 100644 --- a/src/Chaptarr.Api.V1/Books/BookController.cs +++ b/src/Chaptarr.Api.V1/Books/BookController.cs @@ -52,6 +52,8 @@ public class BookController : BookControllerWithSignalR, IHandle, IHandle { + private const int MaxBookshelfAuthorIds = 100; + protected readonly IAuthorService _authorService; protected readonly IEditionService _editionService; protected readonly IAddBookService _addBookService; @@ -1189,6 +1191,58 @@ public ActionResult GetBooksPaged( return Ok(pagedResource); } + [HttpGet("shelf/authors")] + public ActionResult> GetBookshelfAuthors([FromQuery] string mediaType) + { + var parsedMediaType = MediaTypeParameterParser.ParseRequired(mediaType); + var counts = _bookService.GetBookshelfAuthorCounts(parsedMediaType); + + return counts.Select(count => new BookshelfAuthorCountResource + { + AuthorId = count.AuthorId, + BookCount = count.BookCount + }).ToList(); + } + + [HttpGet("shelf")] + public ActionResult> GetBookshelfBooks([FromQuery] List authorIds, [FromQuery] string mediaType) + { + var parsedMediaType = MediaTypeParameterParser.ParseRequired(mediaType); + var distinctAuthorIds = (authorIds ?? new List()).Distinct().ToList(); + + if (distinctAuthorIds.Count == 0 || distinctAuthorIds.Count > MaxBookshelfAuthorIds) + { + return BadRequest($"authorIds must contain between 1 and {MaxBookshelfAuthorIds} author IDs"); + } + + var books = _bookService.GetBooksForBookshelf(distinctAuthorIds, parsedMediaType); + if (books.Count == 0) + { + return new List(); + } + + // The resource mapper reads each book's author; load them once instead of lazily per book. + var authorsById = _authorService.GetAuthors(distinctAuthorIds).ToDictionary(author => author.Id); + foreach (var book in books) + { + if (authorsById.TryGetValue(book.AuthorId, out var author)) + { + book.Author = author; + } + } + + // Statistics for just these authors, so a page of shelf rows doesn't compute them for the whole library. + var apiMediaType = MediaTypeParameterParser.ToApiValue(parsedMediaType); + var authorStatistics = _authorStatisticsService.AuthorStatistics(distinctAuthorIds, apiMediaType); + var statsByBookId = BuildBookStatisticsById(authorStatistics); + + return MapToResource(books, + includeAuthor: false, + statsByBookId, + includeOverview: false, + includeLinks: false); + } + [HttpGet("{id:int}/overview")] public object Overview(int id) { diff --git a/src/Chaptarr.Api.V1/Books/BookshelfAuthorCountResource.cs b/src/Chaptarr.Api.V1/Books/BookshelfAuthorCountResource.cs new file mode 100644 index 00000000..3b0ff8d5 --- /dev/null +++ b/src/Chaptarr.Api.V1/Books/BookshelfAuthorCountResource.cs @@ -0,0 +1,8 @@ +namespace Chaptarr.Api.V1.Books +{ + public class BookshelfAuthorCountResource + { + public int AuthorId { get; set; } + public int BookCount { get; set; } + } +} diff --git a/src/Chaptarr.Core.Test/Api/BookControllerSiblingDeleteInfoFixture.cs b/src/Chaptarr.Core.Test/Api/BookControllerSiblingDeleteInfoFixture.cs index 61bd3d86..0fa49589 100644 --- a/src/Chaptarr.Core.Test/Api/BookControllerSiblingDeleteInfoFixture.cs +++ b/src/Chaptarr.Core.Test/Api/BookControllerSiblingDeleteInfoFixture.cs @@ -87,6 +87,7 @@ public StubAuthorStatisticsService(Func getStats) public AuthorStatistics AuthorStatistics(int authorId) => _getStats?.Invoke(authorId) ?? new AuthorStatistics(); public List AuthorStatistics(string mediaType) => throw new NotImplementedException(); public AuthorStatistics AuthorStatistics(int authorId, string mediaType) => throw new NotImplementedException(); + public List AuthorStatistics(IEnumerable authorIds, string mediaType) => throw new NotImplementedException(); public void InvalidateAuthorCache(int authorId) => throw new NotImplementedException(); } diff --git a/src/Chaptarr.Core.Test/Api/Wanted/WantedMonitoringFilterFixture.cs b/src/Chaptarr.Core.Test/Api/Wanted/WantedMonitoringFilterFixture.cs index 08ad3858..76e580cc 100644 --- a/src/Chaptarr.Core.Test/Api/Wanted/WantedMonitoringFilterFixture.cs +++ b/src/Chaptarr.Core.Test/Api/Wanted/WantedMonitoringFilterFixture.cs @@ -108,6 +108,7 @@ private sealed class StubAuthorStatisticsService : IAuthorStatisticsService public AuthorStatistics AuthorStatistics(int authorId) => new(); public List AuthorStatistics(string mediaType) => new(); public AuthorStatistics AuthorStatistics(int authorId, string mediaType) => new(); + public List AuthorStatistics(IEnumerable authorIds, string mediaType) => new(); public void InvalidateAuthorCache(int authorId) { } } diff --git a/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsImportDeferralFixture.cs b/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsImportDeferralFixture.cs index c4dedc12..e80d4267 100644 --- a/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsImportDeferralFixture.cs +++ b/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsImportDeferralFixture.cs @@ -27,6 +27,7 @@ public List AuthorStatistics() public List AuthorStatistics(int authorId) => Stats(); public List AuthorStatistics(string mediaType) => Stats(); public List AuthorStatistics(int authorId, string mediaType) => Stats(); + public List AuthorStatistics(IEnumerable authorIds, string mediaType) => Stats(); private static List Stats() => new() { new BookStatistics { AuthorId = 1, BookId = 10, BookCount = 1, TotalBookCount = 1 } diff --git a/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsServiceFixture.cs b/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsServiceFixture.cs index e96b6c3b..9376aad8 100644 --- a/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsServiceFixture.cs +++ b/src/Chaptarr.Core.Test/AuthorStats/AuthorStatisticsServiceFixture.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using NUnit.Framework; using NzbDrone.Common.Cache; using NzbDrone.Core.AuthorStats; @@ -24,7 +25,20 @@ public List AuthorStatistics() public List AuthorStatistics(int authorId) => Stats(); public List AuthorStatistics(string mediaType) => Stats(); - public List AuthorStatistics(int authorId, string mediaType) => Stats(); + public List AuthorStatistics(int authorId, string mediaType) + { + SingleAuthorCalls++; + return Stats(); + } + public List> BatchCalls { get; } = new(); + public int SingleAuthorCalls { get; private set; } + + public List AuthorStatistics(IEnumerable authorIds, string mediaType) + { + BatchCalls.Add(new List(authorIds)); + return Stats(); + } + private static List Stats() { return new() @@ -82,5 +96,33 @@ public void file_events_should_evict_the_global_cache_outside_an_import_session( Assert.That(repository.AllAuthorCalls, Is.EqualTo(2)); } + + [Test] + public void batched_author_statistics_should_query_only_uncached_authors_and_share_the_single_author_cache() + { + var repository = new RecordingRepository(); + var service = new AuthorStatisticsService(repository, new CacheManager()); + + var first = service.AuthorStatistics(new[] { 1, 2 }, "audiobook"); + + Assert.That(repository.BatchCalls, Has.Count.EqualTo(1)); + Assert.That(repository.BatchCalls[0], Is.EquivalentTo(new[] { 1, 2 })); + Assert.That(first.Select(statistics => statistics.TotalBookCount).OrderBy(count => count), Is.EqualTo(new[] { 0, 1 })); + + service.AuthorStatistics(new[] { 1, 2 }, "audiobook"); + service.AuthorStatistics(1, "audiobook"); + + Assert.That(repository.BatchCalls, Has.Count.EqualTo(1)); + Assert.That(repository.SingleAuthorCalls, Is.EqualTo(0)); + + service.Handle(new BookFileAddedEvent(new BookFile + { + Author = new NzbDrone.Core.Books.Author { Id = 1 } + })); + service.AuthorStatistics(new[] { 1, 2 }, "audiobook"); + + Assert.That(repository.BatchCalls, Has.Count.EqualTo(2)); + Assert.That(repository.BatchCalls[1], Is.EqualTo(new[] { 1 })); + } } } diff --git a/src/Chaptarr.Core.Test/Books/BookRepositoryIndexFilterFixture.cs b/src/Chaptarr.Core.Test/Books/BookRepositoryIndexFilterFixture.cs index b814959b..ffb88b70 100644 --- a/src/Chaptarr.Core.Test/Books/BookRepositoryIndexFilterFixture.cs +++ b/src/Chaptarr.Core.Test/Books/BookRepositoryIndexFilterFixture.cs @@ -261,6 +261,39 @@ public void find_existing_should_return_partial_results_without_weakening_strict }); } + [Test] + public void bookshelf_author_counts_should_count_each_media_type_separately() + { + WithRepository(sut => + { + var ebookCounts = sut.GetBookCountsByAuthor(BookMediaType.Ebook) + .OrderBy(count => count.AuthorId) + .Select(count => (count.AuthorId, count.BookCount)); + Assert.That(ebookCounts, Is.EqualTo(new[] { (10, 5), (30, 1) })); + + var audiobookCounts = sut.GetBookCountsByAuthor(BookMediaType.Audiobook) + .OrderBy(count => count.AuthorId) + .Select(count => (count.AuthorId, count.BookCount)); + Assert.That(audiobookCounts, Is.EqualTo(new[] { (10, 1), (20, 1) })); + }); + } + + [Test] + public void bookshelf_books_should_return_only_the_requested_authors_and_media_type() + { + WithRepository(sut => + { + Assert.That( + sut.GetBooksByAuthorIds(new[] { 10, 20 }, BookMediaType.Audiobook).Select(book => book.Id).OrderBy(id => id), + Is.EqualTo(new[] { 6, 7 })); + Assert.That( + sut.GetBooksByAuthorIds(new[] { 10, 30 }, BookMediaType.Ebook).Select(book => book.Id).OrderBy(id => id), + Is.EqualTo(new[] { 1, 2, 3, 4, 5, 8 })); + Assert.That(sut.GetBooksByAuthorIds(new[] { 20 }, BookMediaType.Ebook), Is.Empty); + Assert.That(sut.GetBooksByAuthorIds(Array.Empty(), BookMediaType.Ebook), Is.Empty); + }); + } + private static void WithRepository(Action action) { var databasePath = Path.Combine(TestContext.CurrentContext.WorkDirectory, $"book_index_filter_{Guid.NewGuid():N}.db"); diff --git a/src/Chaptarr.Core.Test/Books/BookServiceGetBooksLoadsEditionsFixture.cs b/src/Chaptarr.Core.Test/Books/BookServiceGetBooksLoadsEditionsFixture.cs index 39833b68..146f8a57 100644 --- a/src/Chaptarr.Core.Test/Books/BookServiceGetBooksLoadsEditionsFixture.cs +++ b/src/Chaptarr.Core.Test/Books/BookServiceGetBooksLoadsEditionsFixture.cs @@ -11,6 +11,7 @@ using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Profiles.Qualities; using NzbDrone.Core.Qualities; +using NzbDrone.Core.RootFolders; namespace Chaptarr.Core.Test.Books { @@ -100,6 +101,21 @@ public IEnumerable FindExisting(IEnumerable ids) public void UpdateMonitoringByAuthorAndMediaType(int authorId, BookMediaType mediaType, bool monitored, IEnumerable exceptBookIds = null) => throw new NotImplementedException(); public BookBucketResource GetBookBuckets(string sortKey, string sortDirection, bool includeUnmonitored = false, string mediaType = null, bool? downloaded = null) => throw new NotImplementedException(); public PagedBookResource GetBooksPaged(int offset, int pageSize, string sortKey, string sortDirection, bool includeUnmonitored = false, string mediaType = null, bool? downloaded = null) => throw new NotImplementedException(); + + public List GetBooksByAuthorIds(IEnumerable authorIds, BookMediaType mediaType) + { + var authorIdSet = authorIds.ToHashSet(); + return _booksById.Values.Where(b => authorIdSet.Contains(b.AuthorId) && b.MediaType == mediaType).ToList(); + } + + public List GetBookCountsByAuthor(BookMediaType mediaType) + { + return _booksById.Values + .Where(b => b.MediaType == mediaType) + .GroupBy(b => b.AuthorId) + .Select(g => new AuthorBookCount { AuthorId = g.Key, BookCount = g.Count() }) + .ToList(); + } } private sealed class StubEditionService : IEditionService @@ -140,6 +156,28 @@ public List GetEditionsByBook(IEnumerable bookIds) public List SetMonitored(Edition edition, bool isManualSelection = false) => throw new NotImplementedException(); } + private sealed class StubRootFolderService : IRootFolderService + { + private readonly List _rootFolders; + + public StubRootFolderService(params RootFolder[] rootFolders) + { + _rootFolders = rootFolders.ToList(); + } + + public List All() => _rootFolders; + public List AllWithSpaceStats() => throw new NotImplementedException(); + public RootFolder Add(RootFolder rootFolder) => throw new NotImplementedException(); + public RootFolder Update(RootFolder rootFolder) => throw new NotImplementedException(); + public void Remove(int id) => throw new NotImplementedException(); + public RootFolder Get(int id) => throw new NotImplementedException(); + public List AllForTag(int tagId) => throw new NotImplementedException(); + public RootFolder GetBestRootFolder(string path) => throw new NotImplementedException(); + public RootFolder GetBestRootFolder(string path, List allRootFolders) => throw new NotImplementedException(); + public string GetBestRootFolderPath(string path) => throw new NotImplementedException(); + public string GetBestRootFolderPath(string path, List allRootFolders) => throw new NotImplementedException(); + } + private sealed class StubAuthorService : IAuthorService { private readonly Dictionary _authors; @@ -350,5 +388,65 @@ public void should_batch_load_files_for_book_in_edition_iteration_order() Assert.That(result.Editions[1].BookFiles.Select(file => file.Id), Is.EqualTo(new[] { 300 })); Assert.That(result.Editions[2].BookFiles, Is.Null); } + + [TestCase(BookMediaType.Audiobook, 1, 10)] + [TestCase(BookMediaType.Ebook, 2, 20)] + public void bookshelf_books_should_carry_only_monitored_editions_of_the_requested_media_type(BookMediaType mediaType, int expectedBookId, int expectedEditionId) + { + var service = CreateBookshelfService(new RootFolder { FolderType = FolderType.Mixed }); + + var results = service.GetBooksForBookshelf(new[] { 10, 99 }, mediaType); + + Assert.That(results.Select(book => book.Id), Is.EqualTo(new[] { expectedBookId })); + Assert.That(results[0].Editions.Select(edition => edition.Id), Is.EqualTo(new[] { expectedEditionId })); + Assert.That(results[0].BookFiles, Is.Empty); + } + + [TestCase(BookMediaType.Audiobook, FolderType.Ebook)] + [TestCase(BookMediaType.Ebook, FolderType.Audiobook)] + public void bookshelf_should_be_empty_without_a_root_folder_for_the_media_type(BookMediaType mediaType, FolderType otherFolderType) + { + var service = CreateBookshelfService(new RootFolder { FolderType = otherFolderType }); + + Assert.That(service.GetBookshelfAuthorCounts(mediaType), Is.Empty); + Assert.That(service.GetBooksForBookshelf(new[] { 10 }, mediaType), Is.Empty); + } + + [Test] + public void bookshelf_author_counts_should_be_scoped_to_the_media_type() + { + var service = CreateBookshelfService(new RootFolder { FolderType = FolderType.Mixed }); + + var counts = service.GetBookshelfAuthorCounts(BookMediaType.Ebook); + + Assert.That(counts.Select(count => (count.AuthorId, count.BookCount)), Is.EqualTo(new[] { (10, 1) })); + } + + private static BookService CreateBookshelfService(params RootFolder[] rootFolders) + { + var author = new Author { Id = 10, Name = "Isaac Asimov" }; + var books = new[] + { + new Book { Id = 1, AuthorId = 10, Title = "Foundation", MediaType = BookMediaType.Audiobook }, + new Book { Id = 2, AuthorId = 10, Title = "Foundation", MediaType = BookMediaType.Ebook } + }; + var editions = new[] + { + new Edition { Id = 10, BookId = 1, Title = "Foundation", Monitored = true }, + new Edition { Id = 11, BookId = 1, Title = "Foundation", Monitored = false }, + new Edition { Id = 20, BookId = 2, Title = "Foundation", Monitored = true } + }; + + return new BookService( + new StubBookRepository(books), + new StubEditionService(editions), + eventAggregator: null, + authorService: new StubAuthorService(new[] { author }), + mediaFileService: null, + rootFolderService: new StubRootFolderService(rootFolders), + new StubSeriesBookLinkRepository(), + multiCopySeriesService: null, + logger: LogManager.GetCurrentClassLogger()); + } } } diff --git a/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs b/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs index 74f1b1f1..f3e00d7d 100644 --- a/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs +++ b/src/NzbDrone.Core/AuthorStats/AuthorStatisticsRepository.cs @@ -13,6 +13,7 @@ public interface IAuthorStatisticsRepository List AuthorStatistics(int authorId); List AuthorStatistics(string mediaType); List AuthorStatistics(int authorId, string mediaType); + List AuthorStatistics(IEnumerable authorIds, string mediaType); } public class AuthorStatisticsRepository : IAuthorStatisticsRepository @@ -53,6 +54,19 @@ public List AuthorStatistics(int authorId, string mediaType) return Query(builder); } + public List AuthorStatistics(IEnumerable authorIds, string mediaType) + { + var idsArray = authorIds?.Distinct().ToArray() ?? Array.Empty(); + if (idsArray.Length == 0) + { + return new List(); + } + + var builder = Builder(_database.DatabaseType).Where(author => Enumerable.Contains(idsArray, author.Id)); + ApplyMediaTypeFilter(builder, mediaType); + return Query(builder); + } + internal static string BuildBaseSql(DatabaseType databaseType) { return Builder(databaseType).AddTemplate(_selectTemplate).RawSql; diff --git a/src/NzbDrone.Core/AuthorStats/AuthorStatisticsService.cs b/src/NzbDrone.Core/AuthorStats/AuthorStatisticsService.cs index 5a3b796a..887cf1d2 100644 --- a/src/NzbDrone.Core/AuthorStats/AuthorStatisticsService.cs +++ b/src/NzbDrone.Core/AuthorStats/AuthorStatisticsService.cs @@ -15,6 +15,7 @@ public interface IAuthorStatisticsService AuthorStatistics AuthorStatistics(int authorId); List AuthorStatistics(string mediaType); AuthorStatistics AuthorStatistics(int authorId, string mediaType); + List AuthorStatistics(IEnumerable authorIds, string mediaType); void InvalidateAuthorCache(int authorId); } @@ -81,7 +82,7 @@ public List AuthorStatistics(string mediaType) public AuthorStatistics AuthorStatistics(int authorId, string mediaType) { - var cacheKey = string.IsNullOrEmpty(mediaType) ? authorId.ToString() : $"{authorId}_{mediaType}"; + var cacheKey = GetAuthorCacheKey(authorId, mediaType); var statistics = _cache.Get(cacheKey, () => _authorStatisticsRepository.AuthorStatistics(authorId, mediaType)); if (statistics == null || statistics.Count == 0) @@ -92,6 +93,54 @@ public AuthorStatistics AuthorStatistics(int authorId, string mediaType) return MapAuthorStatistics(statistics); } + // One query for every author that isn't cached yet, stored under the same keys as the single-author lookup. + public List AuthorStatistics(IEnumerable authorIds, string mediaType) + { + var results = new List(); + var uncachedAuthorIds = new List(); + + foreach (var authorId in authorIds.Distinct()) + { + var cached = _cache.Find(GetAuthorCacheKey(authorId, mediaType)); + + if (cached == null) + { + uncachedAuthorIds.Add(authorId); + } + else + { + results.Add(cached.Count == 0 ? new AuthorStatistics() : MapAuthorStatistics(cached)); + } + } + + if (uncachedAuthorIds.Count == 0) + { + return results; + } + + var statisticsByAuthor = _authorStatisticsRepository.AuthorStatistics(uncachedAuthorIds, mediaType) + .GroupBy(statistics => statistics.AuthorId) + .ToDictionary(group => group.Key, group => group.ToList()); + + foreach (var authorId in uncachedAuthorIds) + { + if (!statisticsByAuthor.TryGetValue(authorId, out var statistics)) + { + statistics = new List(); + } + + _cache.Set(GetAuthorCacheKey(authorId, mediaType), statistics); + results.Add(statistics.Count == 0 ? new AuthorStatistics() : MapAuthorStatistics(statistics)); + } + + return results; + } + + private static string GetAuthorCacheKey(int authorId, string mediaType) + { + return string.IsNullOrEmpty(mediaType) ? authorId.ToString() : $"{authorId}_{mediaType}"; + } + private static AuthorStatistics MapAuthorStatistics(List bookStatistics) { return new AuthorStatistics diff --git a/src/NzbDrone.Core/Books/Repositories/BookRepository.cs b/src/NzbDrone.Core/Books/Repositories/BookRepository.cs index 50726452..ceddc207 100644 --- a/src/NzbDrone.Core/Books/Repositories/BookRepository.cs +++ b/src/NzbDrone.Core/Books/Repositories/BookRepository.cs @@ -18,6 +18,12 @@ public sealed class BookSearchTarget public int AuthorId { get; set; } } + public sealed class AuthorBookCount + { + public int AuthorId { get; set; } + public int BookCount { get; set; } + } + public interface IBookRepository : IBasicRepository { List GetBooks(int authorId); @@ -26,6 +32,8 @@ public interface IBookRepository : IBasicRepository List GetLastBooks(IEnumerable authorIds); List GetNextBooks(IEnumerable authorIds); List GetBooksByAuthorId(int authorId); + List GetBooksByAuthorIds(IEnumerable authorIds, BookMediaType mediaType) => throw new NotImplementedException(); + List GetBookCountsByAuthor(BookMediaType mediaType) => throw new NotImplementedException(); List GetBooksForRefresh(int authorId, IEnumerable providerIds); List GetBooksByFileIds(IEnumerable fileIds); Book FindByTitle(int authorId, string title); @@ -198,6 +206,26 @@ public List GetBooksByAuthorId(int authorId) return Query(s => s.AuthorId == authorId); } + public List GetBooksByAuthorIds(IEnumerable authorIds, BookMediaType mediaType) + { + var idsArray = authorIds?.Distinct().ToArray() ?? Array.Empty(); + if (idsArray.Length == 0) + { + return new List(); + } + + return Query(Builder().Where(x => Enumerable.Contains(idsArray, x.AuthorId) && x.MediaType == mediaType)); + } + + public List GetBookCountsByAuthor(BookMediaType mediaType) + { + var sql = $"SELECT \"AuthorId\", COUNT(*) AS \"BookCount\" FROM \"{_table}\" WHERE \"MediaType\" = @mediaType GROUP BY \"AuthorId\""; + using (var conn = _database.OpenConnection()) + { + return conn.Query(sql, new { mediaType = (int)mediaType }).ToList(); + } + } + public List GetBooksForRefresh(int authorId, IEnumerable providerIds) { // Refresh must be author-scoped. Provider-ID lookups are handled downstream by the diff --git a/src/NzbDrone.Core/Books/Services/BookService.cs b/src/NzbDrone.Core/Books/Services/BookService.cs index 39f75f8e..1745de61 100644 --- a/src/NzbDrone.Core/Books/Services/BookService.cs +++ b/src/NzbDrone.Core/Books/Services/BookService.cs @@ -81,6 +81,8 @@ void RefreshProviderAliases(Book book) { } PagedBookResource GetBooksPaged(int offset, int pageSize, string sortKey, string sortDirection, bool includeUnmonitored = false, string mediaType = null, bool? downloaded = null); PagedBookResource GetBooksPaged(int offset, int pageSize, string sortKey, string sortDirection, bool includeUnmonitored, string mediaType, bool? downloaded, bool? monitored, bool? missing = null, bool? wanted = null) => GetBooksPaged(offset, pageSize, sortKey, sortDirection, includeUnmonitored, mediaType, downloaded); List GetBookIds(bool includeUnmonitored = false, string mediaType = null, bool? downloaded = null, bool? monitored = null, bool? missing = null, bool? wanted = null) => throw new NotImplementedException(); + List GetBookshelfAuthorCounts(BookMediaType mediaType) => throw new NotImplementedException(); + List GetBooksForBookshelf(IEnumerable authorIds, BookMediaType mediaType) => throw new NotImplementedException(); } public class BookService : IBookService, @@ -2244,12 +2246,7 @@ public List GetBooksForDisplay(int? authorId = null, string mediaType = nu // CRITICAL: Check if root folder exists for this media type // If no root folder, return empty list (GUI shows nothing) - var rootFolders = _rootFolderService.All(); - var hasRootFolderForType = targetMediaType == BookMediaType.Audiobook - ? rootFolders.Any(rf => rf.FolderType == FolderType.Audiobook || rf.FolderType == FolderType.Mixed) - : rootFolders.Any(rf => rf.FolderType == FolderType.Ebook || rf.FolderType == FolderType.Mixed); - - if (!hasRootFolderForType) + if (!HasRootFolderForMediaType(targetMediaType)) { _logger.Debug("[DISPLAY-DEBUG] No root folder configured for {0}, returning empty list", mediaType); return new List(); @@ -2308,6 +2305,69 @@ public List GetBooksForDisplay(int? authorId = null, string mediaType = nu return allBooks; } + public List GetBookshelfAuthorCounts(BookMediaType mediaType) + { + if (!HasRootFolderForMediaType(mediaType)) + { + return new List(); + } + + return _bookRepository.GetBookCountsByAuthor(mediaType); + } + + public List GetBooksForBookshelf(IEnumerable authorIds, BookMediaType mediaType) + { + if (!HasRootFolderForMediaType(mediaType)) + { + return new List(); + } + + var books = _bookRepository.GetBooksByAuthorIds(authorIds, mediaType); + if (!books.Any()) + { + return books; + } + + // Same lean shape as the general book list: monitored editions only, no files. + var bookIds = books.Select(b => b.Id).ToList(); + var monitoredEditions = _editionService.GetEditionsByBook(bookIds) + .Where(e => e.Monitored) + .ToList(); + var editionsByBook = monitoredEditions + .GroupBy(e => e.BookId) + .ToDictionary(g => g.Key, g => g.ToList()); + + foreach (var book in books) + { + book.BookFiles = new List(); + + if (editionsByBook.TryGetValue(book.Id, out var bookEditions)) + { + foreach (var edition in bookEditions) + { + edition.BookFiles = null; + } + + book.Editions = bookEditions; + } + else + { + book.Editions = new List(); + } + } + + return books; + } + + private bool HasRootFolderForMediaType(BookMediaType mediaType) + { + var rootFolders = _rootFolderService.All(); + + return mediaType == BookMediaType.Audiobook + ? rootFolders.Any(rf => rf.FolderType == FolderType.Audiobook || rf.FolderType == FolderType.Mixed) + : rootFolders.Any(rf => rf.FolderType == FolderType.Ebook || rf.FolderType == FolderType.Mixed); + } + public List GetBooksByBaseId(string baseBookId) { // Find all books that share any provider ID