From 65ef58cfe89b7808d7a373e81576b34f3b2f3947 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Sun, 5 Jul 2026 20:11:26 +0800 Subject: [PATCH 1/5] feat: unified Transformers4Rec with CLM/MLM/PLM/RTD training objective --- README.md | 3 +- cornac/models/__init__.py | 3 +- cornac/models/seq_utils/__init__.py | 6 +- cornac/models/seq_utils/iterators.py | 99 +++- cornac/models/transformer_rec/__init__.py | 22 + cornac/models/transformer_rec/backbones.py | 135 +++++ .../transformer_rec/objectives/__init__.py | 29 ++ .../models/transformer_rec/objectives/base.py | 91 ++++ .../models/transformer_rec/objectives/clm.py | 95 ++++ .../models/transformer_rec/objectives/mlm.py | 82 +++ .../models/transformer_rec/objectives/plm.py | 158 ++++++ .../models/transformer_rec/objectives/rtd.py | 171 ++++++ .../transformer_rec/recom_transformer_rec.py | 493 ++++++++++++++++++ .../models/transformer_rec/requirements.txt | 2 + .../models/transformer_rec/transformer_rec.py | 183 +++++++ examples/transformer_rec_diginetica.py | 66 +-- 16 files changed, 1599 insertions(+), 39 deletions(-) create mode 100644 cornac/models/transformer_rec/__init__.py create mode 100644 cornac/models/transformer_rec/backbones.py create mode 100644 cornac/models/transformer_rec/objectives/__init__.py create mode 100644 cornac/models/transformer_rec/objectives/base.py create mode 100644 cornac/models/transformer_rec/objectives/clm.py create mode 100644 cornac/models/transformer_rec/objectives/mlm.py create mode 100644 cornac/models/transformer_rec/objectives/plm.py create mode 100644 cornac/models/transformer_rec/objectives/rtd.py create mode 100644 cornac/models/transformer_rec/recom_transformer_rec.py create mode 100644 cornac/models/transformer_rec/requirements.txt create mode 100644 cornac/models/transformer_rec/transformer_rec.py diff --git a/README.md b/README.md index 6f71db14..b24439d5 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ The table below lists the recommendation models/algorithms featured in Cornac. E | 2023 | [Scalable Approximate NonSymmetric Autoencoder (SANSA)](cornac/models/sansa), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.sansa.recom_sansa), [paper](https://dl.acm.org/doi/10.1145/3604915.3608827) | Collaborative Filtering | [requirements](cornac/models/sansa/requirements.txt), CPU | [quick-start](examples/sansa_movielens.py), [150k-items](examples/sansa_tradesy.py) | 2022 | [Disentangled Multimodal Representation Learning for Recommendation (DMRL)](cornac/models/dmrl), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.dmrl.recom_dmrl), [paper](https://arxiv.org/pdf/2203.05406.pdf) | Content-Based / Text & Image | [requirements](cornac/models/dmrl/requirements.txt), CPU / GPU | [quick-start](examples/dmrl_example.py) | 2021 | [Bilateral Variational Autoencoder for Collaborative Filtering (BiVAECF)](cornac/models/bivaecf), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.bivaecf.recom_bivaecf), [paper](https://dl.acm.org/doi/pdf/10.1145/3437963.3441759) | Collaborative Filtering / Content-Based | [requirements](cornac/models/bivaecf/requirements.txt), CPU / GPU | [quick-start](https://github.com/PreferredAI/bi-vae), [deep-dive](https://github.com/recommenders-team/recommenders/blob/main/examples/02_model_collaborative_filtering/cornac_bivae_deep_dive.ipynb) -| | [GPT-2 for Sequential Recommendation (GPT2Rec)](cornac/models/gpt2rec), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.gpt2rec.recom_gpt2rec), [paper](https://dl.acm.org/doi/10.1145/3460231.3474255) | Next-Item | [requirements](cornac/models/gpt2rec/requirements.txt), CPU / GPU | [quick-start](examples/transformer_rec_diginetica.py) +| | [Transformers4Rec-style Unified Transformer Recommender (TransformerRec)](cornac/models/transformer_rec), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.transformer_rec.recom_transformer_rec), [paper](https://dl.acm.org/doi/10.1145/3460231.3474255) | Next-Item | [requirements](cornac/models/transformer_rec/requirements.txt), CPU / GPU | [quick-start](examples/transformer_rec_diginetica.py) | | [Causal Inference for Visual Debiasing in Visually-Aware Recommendation (CausalRec)](cornac/models/causalrec), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.causalrec.recom_causalrec), [paper](https://arxiv.org/abs/2107.02390) | Content-Based / Image | [requirements](cornac/models/causalrec/requirements.txt), CPU / GPU | [quick-start](examples/causalrec_clothing.py) | | [Explainable Recommendation with Comparative Constraints on Product Aspects (ComparER)](cornac/models/comparer), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.comparer.recom_comparer_sub), [paper](https://dl.acm.org/doi/pdf/10.1145/3437963.3441754) | Explainable | CPU | [quick-start](https://github.com/PreferredAI/ComparER) | 2020 | [Adversarial Multimedia Recommendation (AMR)](cornac/models/amr), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.amr.recom_amr), [paper](https://ieeexplore.ieee.org/document/8618394) | Content-Based / Image | [requirements](cornac/models/amr/requirements.txt), CPU / GPU | [quick-start](examples/amr_clothing.py) @@ -166,7 +166,6 @@ The table below lists the recommendation models/algorithms featured in Cornac. E | | [Temporal-Item-Frequency-based User-KNN (TIFUKNN)](cornac/models/tifuknn), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.tifuknn.recom_tifuknn), [paper](https://arxiv.org/pdf/2006.00556.pdf) | Next-Basket | CPU | [quick-start](examples/tifuknn_tafeng.py) | | [Variational Autoencoder for Top-N Recommendations (RecVAE)](cornac/models/recvae), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.recvae.recom_recvae), [paper](https://doi.org/10.1145/3336191.3371831) | Collaborative Filtering | [requirements](cornac/models/recvae/requirements.txt), CPU / GPU | [quick-start](examples/recvae_example.py) | 2019 | [Correlation-Sensitive Next-Basket Recommendation (Beacon)](cornac/models/beacon), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#correlation-sensitive-next-basket-recommendation-beacon), [paper](https://www.ijcai.org/proceedings/2019/0389.pdf) | Next-Basket | [requirements](cornac/models/beacon/requirements.txt), CPU / GPU | [quick-start](examples/beacon_tafeng.py) -| | [BERT4Rec: Sequential Recommendation with Bidirectional Encoder Representations from Transformer (BERT4Rec)](cornac/models/bert4rec), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.bert4rec.recom_bert4rec), [paper](https://arxiv.org/pdf/1904.06690.pdf) | Next-Item | [requirements](cornac/models/bert4rec/requirements.txt), CPU / GPU | [quick-start](examples/transformer_rec_diginetica.py) | | [Embarrassingly Shallow Autoencoders for Sparse Data (EASEᴿ)](cornac/models/ease), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.ease.recom_ease), [paper](https://arxiv.org/pdf/1905.03375.pdf) | Collaborative Filtering | CPU | [quick-start](examples/ease_movielens.py) | | [Neural Graph Collaborative Filtering (NGCF)](cornac/models/ngcf), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.ngcf.recom_ngcf), [paper](https://arxiv.org/pdf/1905.08108.pdf) | Collaborative Filtering | [requirements](cornac/models/ngcf/requirements.txt), CPU / GPU | [quick-start](examples/ngcf_example.py) | | [Sampler Design for Bayesian Personalized Ranking by Leveraging View Data (VEBPR)](cornac/models/bpr), [paper](https://arxiv.org/pdf/1809.08162) | Collaborative Filtering | CPU | [quick-start](examples/vebpr_example.py) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 52ad564e..50f6c145 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -24,7 +24,6 @@ from .ann import ScaNNANN from .baseline_only import BaselineOnly from .beacon import Beacon -from .bert4rec import BERT4Rec from .bivaecf import BiVAECF from .bpr import BPR from .bpr import WBPR @@ -50,7 +49,6 @@ from .gcmc import GCMC from .global_avg import GlobalAvg from .gp_top import GPTop -from .gpt2rec import GPT2Rec from .gru4rec import GRU4Rec from .hft import HFT from .hpf import HPF @@ -84,6 +82,7 @@ from .spop import SPop from .svd import SVD from .tifuknn import TIFUKNN +from .transformer_rec import TransformerRec from .trirank import TriRank from .upcf import UPCF from .vaecf import VAECF diff --git a/cornac/models/seq_utils/__init__.py b/cornac/models/seq_utils/__init__.py index da8cf33c..4f06b653 100644 --- a/cornac/models/seq_utils/__init__.py +++ b/cornac/models/seq_utils/__init__.py @@ -29,14 +29,18 @@ -- per-item RNN, session-based. ``session_seq_iter`` ``(in_uids, hist_iids, out_iids)`` -- sequence models, session-based. + ``padded_session_iter`` ``(uids, padded_seqs)`` -- whole-session + transformer models. ========================= ================================================= """ -from .iterators import io_iter, session_seq_iter +from .iterators import build_neg_sampler, io_iter, padded_session_iter, session_seq_iter from .selection import val_score __all__ = [ "io_iter", "session_seq_iter", + "padded_session_iter", + "build_neg_sampler", "val_score", ] diff --git a/cornac/models/seq_utils/iterators.py b/cornac/models/seq_utils/iterators.py index 5e23a4c8..6ca82c37 100644 --- a/cornac/models/seq_utils/iterators.py +++ b/cornac/models/seq_utils/iterators.py @@ -23,8 +23,24 @@ from ...utils.common import get_rng -def _build_neg_sampler(uir_tuple, sample_alpha): - """Precompute popularity-based sampling distribution over items.""" +def build_neg_sampler(uir_tuple, sample_alpha): + """Precompute a popularity-based sampling distribution over items. + + Parameters + ---------- + uir_tuple : tuple + ``(user_ids, item_ids, ratings)`` arrays; only ``item_ids`` is used. + sample_alpha : float + Popularity smoothing exponent. ``0`` gives a uniform distribution + (over observed items) and ``1`` gives raw popularity weighting. + + Returns + ------- + item_indices : numpy.ndarray, shape (num_items,), dtype int + Item ids ordered from most to least popular. + item_dist : numpy.ndarray, shape (num_items,), dtype float + Sampling probabilities aligned with ``item_indices`` (sums to 1). + """ item_count = Counter(uir_tuple[1]) item_indices = np.array([iid for iid, _ in item_count.most_common()], dtype="int") item_dist = np.array([cnt for _, cnt in item_count.most_common()], dtype="float") ** sample_alpha @@ -59,7 +75,7 @@ def io_iter(s_iter, uir_tuple, n_sample=0, sample_alpha=0, rng=None, batch_size= c_pool = [None for _ in range(batch_size)] sizes = np.zeros(batch_size, dtype="int") if n_sample > 0: - item_indices, item_dist = _build_neg_sampler(uir_tuple, sample_alpha) + item_indices, item_dist = build_neg_sampler(uir_tuple, sample_alpha) for _, batch_mapped_ids in s_iter(batch_size, shuffle): l_pool += batch_mapped_ids @@ -144,7 +160,7 @@ def session_seq_iter( if shuffle: rng.shuffle(sids) if n_sample > 0: - item_indices, item_dist = _build_neg_sampler(uir_tuple, sample_alpha) + item_indices, item_dist = build_neg_sampler(uir_tuple, sample_alpha) buffer_uids, buffer_hist, buffer_target = [], [], [] for sid in sids: @@ -184,3 +200,78 @@ def session_seq_iter( np.array(buffer_hist, dtype="int"), out_iids, ) + + +def padded_session_iter( + train_set, + pad_index, + batch_size=64, + max_len=20, + rng=None, + shuffle=True, +): + """Whole-session iterator for transformer next-item models. + + Iterates over sessions, yielding each session as ONE left-padded row + (no prefix breakdown, no target split, no negative sampling). Training + objectives (CLM/MLM/PLM/RTD) derive their own targets from the raw + padded sessions downstream. + + For a session ``[i0, i1, ..., iT]`` the row keeps the last ``max_len`` + items (the head is truncated when longer) and is left-padded with + ``pad_index`` to exactly ``max_len``. Sessions with fewer than 2 items + are skipped. + + Parameters + ---------- + train_set : :class:`~cornac.data.SequentialDataset` + Must expose ``uir_tuple`` and ``sessions``. + pad_index : int + Padding token used to left-pad short sessions. + batch_size : int, default 64 + Number of sessions per yielded batch. + max_len : int, default 20 + Fixed sequence length of every row (truncate head / left-pad). + rng : numpy.random.RandomState, optional + Random state used to shuffle session order. Defaults to a fresh one. + shuffle : bool, default True + Whether to shuffle the session order each epoch. + + Yields + ------ + uids : numpy.ndarray, shape (B,), dtype int + User id of each session in the batch. + padded_seqs : numpy.ndarray, shape (B, max_len), dtype int + Left-padded, head-truncated item-id sequences. ``B == batch_size`` + for full batches; the final partial batch is yielded whenever it is + non-empty (``B >= 1``). + """ + rng = rng if rng is not None else get_rng(None) + uir_tuple = train_set.uir_tuple + sessions = train_set.sessions + sids = list(sessions.keys()) + if shuffle: + rng.shuffle(sids) + + buffer_uids, buffer_seqs = [], [] + for sid in sids: + mapped_ids = sessions[sid] + items = list(uir_tuple[1][mapped_ids]) + if len(items) < 2: + continue + uid = int(uir_tuple[0][mapped_ids[0]]) + seq = items[-max_len:] + seq = [pad_index] * (max_len - len(seq)) + list(seq) + buffer_uids.append(uid) + buffer_seqs.append(seq) + if len(buffer_uids) == batch_size: + yield ( + np.array(buffer_uids, dtype="int"), + np.array(buffer_seqs, dtype="int"), + ) + buffer_uids, buffer_seqs = [], [] + if len(buffer_uids) >= 1: + yield ( + np.array(buffer_uids, dtype="int"), + np.array(buffer_seqs, dtype="int"), + ) diff --git a/cornac/models/transformer_rec/__init__.py b/cornac/models/transformer_rec/__init__.py new file mode 100644 index 00000000..0bfa604f --- /dev/null +++ b/cornac/models/transformer_rec/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""TransformerRec: a unified HuggingFace-backbone next-item recommender. + +Supports interchangeable transformer backbones +Supported models: ``bert``/``gpt2``/``xlnet``/``electra`` +Supported training objectives: ``CLM``/``MLM``/``PLM``/``RTD`` over a single shared item-embedding scoring head. +""" + +from .recom_transformer_rec import TransformerRec diff --git a/cornac/models/transformer_rec/backbones.py b/cornac/models/transformer_rec/backbones.py new file mode 100644 index 00000000..2614e6bf --- /dev/null +++ b/cornac/models/transformer_rec/backbones.py @@ -0,0 +1,135 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""HuggingFace backbone registry for :class:`TransformerRecModel`. + +Each backbone is a bare HuggingFace transformer encoder that consumes +``inputs_embeds`` + ``attention_mask`` and exposes ``.last_hidden_state`` on +its output. +""" + + +def build_bert(vocab_size, embedding_dim, max_len, n_layers, n_heads, dropout, pad_idx): + """Build a bidirectional BERT encoder (see BERT4Rec parameterization).""" + from transformers.models.bert import BertConfig, BertModel + + config = BertConfig( + vocab_size=vocab_size, + hidden_size=embedding_dim, + num_hidden_layers=n_layers, + num_attention_heads=n_heads, + intermediate_size=embedding_dim * 4, + hidden_act="gelu", + hidden_dropout_prob=dropout, + attention_probs_dropout_prob=dropout, + max_position_embeddings=max_len + 1, + pad_token_id=pad_idx, + layer_norm_eps=1e-12, + use_cache=False, + ) + return BertModel(config) + + +def build_gpt2(vocab_size, embedding_dim, max_len, n_layers, n_heads, dropout, pad_idx): + """Build a causal GPT-2 decoder (see GPT2Rec parameterization).""" + from transformers.models.gpt2 import GPT2Config, GPT2Model + + config = GPT2Config( + vocab_size=vocab_size, + n_positions=max_len + 1, + n_embd=embedding_dim, + n_layer=n_layers, + n_head=n_heads, + n_inner=embedding_dim * 4, + activation_function="gelu_new", + resid_pdrop=dropout, + embd_pdrop=dropout, + attn_pdrop=dropout, + pad_token_id=pad_idx, + layer_norm_epsilon=1e-12, + use_cache=False, + ) + return GPT2Model(config) + + +def build_xlnet(vocab_size, embedding_dim, max_len, n_layers, n_heads, dropout, pad_idx): + """Build an XLNet encoder (two-stream; supports perm_mask/target_mapping). + + ``d_head`` is set to ``embedding_dim // n_heads`` so that the attention + output width matches ``d_model`` (the standard XLNet convention). + """ + from transformers.models.xlnet import XLNetConfig, XLNetModel + + config = XLNetConfig( + vocab_size=vocab_size, + d_model=embedding_dim, + n_layer=n_layers, + n_head=n_heads, + d_head=max(1, embedding_dim // n_heads), + d_inner=embedding_dim * 4, + ff_activation="gelu", + dropout=dropout, + pad_token_id=pad_idx, + layer_norm_eps=1e-12, + use_mems_train=False, + use_mems_eval=False, + ) + return XLNetModel(config) + + +def build_electra(vocab_size, embedding_dim, max_len, n_layers, n_heads, dropout, pad_idx): + """Build an ELECTRA encoder (embedding_size and hidden_size both = dim).""" + from transformers.models.electra import ElectraConfig, ElectraModel + + config = ElectraConfig( + vocab_size=vocab_size, + embedding_size=embedding_dim, + hidden_size=embedding_dim, + num_hidden_layers=n_layers, + num_attention_heads=n_heads, + intermediate_size=embedding_dim * 4, + hidden_act="gelu", + hidden_dropout_prob=dropout, + attention_probs_dropout_prob=dropout, + max_position_embeddings=max_len + 1, + pad_token_id=pad_idx, + layer_norm_eps=1e-12, + use_cache=False, + ) + return ElectraModel(config) + + +# name -> (build_fn, attention_type in {"causal", "bidirectional"}) +BACKBONES = { + "bert": (build_bert, "bidirectional"), + "gpt2": (build_gpt2, "causal"), + "xlnet": (build_xlnet, "bidirectional"), + "electra": (build_electra, "bidirectional"), +} + +# Read-only view mapping backbone name -> attention type (for validity checks). +ATTENTION_TYPES = {name: attn for name, (_, attn) in BACKBONES.items()} + + +def get_backbone(name): + """Return the ``(build_fn, attention_type)`` pair for ``name``. + + Raises + ------ + ValueError + If ``name`` is not a registered backbone. + """ + if name not in BACKBONES: + raise ValueError(f"Unknown backbone '{name}'. Supported: {sorted(BACKBONES)}") + return BACKBONES[name] diff --git a/cornac/models/transformer_rec/objectives/__init__.py b/cornac/models/transformer_rec/objectives/__init__.py new file mode 100644 index 00000000..1a21f886 --- /dev/null +++ b/cornac/models/transformer_rec/objectives/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Training objectives for :class:`TransformerRecModel`.""" + +from .base import Objective +from .clm import CLMObjective +from .mlm import MLMObjective +from .plm import PLMObjective +from .rtd import RTDObjective + +__all__ = [ + "Objective", + "CLMObjective", + "MLMObjective", + "PLMObjective", + "RTDObjective", +] diff --git a/cornac/models/transformer_rec/objectives/base.py b/cornac/models/transformer_rec/objectives/base.py new file mode 100644 index 00000000..1135974d --- /dev/null +++ b/cornac/models/transformer_rec/objectives/base.py @@ -0,0 +1,91 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Training-objective contract for :class:`TransformerRecModel`. + +An :class:`Objective` turns left-padded whole sessions into inputs, loss +positions, and targets; runs the model; and returns a scalar loss computed +by one of the shared :mod:`cornac.models.seq_utils` loss functions against a +``(M, M + N)`` score matrix. +""" + + +class Objective: + """Base training objective (next-item / causal defaults).""" + + #: Backbone attention types this objective is compatible with. + VALID_ATTENTION = ("causal", "bidirectional") + #: Whether the objective feeds the ``mask_idx`` token to the model. + uses_mask_token = False + + def __init__(self, pad_idx, mask_idx, rng): + self.pad_idx, self.mask_idx, self.rng = pad_idx, mask_idx, rng + + def build(self, model, device): + """One-time hook after :class:`TransformerRecModel` construction. + + Objectives that need extra sub-modules (e.g. RTD builds its generator + here) create them in this hook. Default: no-op. + """ + + def parameters(self): + """Extra trainable parameters beyond the model's. Default: ``[]``.""" + return [] + + def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): + """Compute the training loss for one batch of sessions. + + Parameters + ---------- + model : TransformerRecModel + seqs : numpy.ndarray, shape (B, T) + Left-padded whole sessions. + sample_negatives : callable or None + ``n -> numpy.ndarray`` returning ``n`` sampled negative item ids. + loss_fn : callable + A loss from :mod:`cornac.models.seq_utils.losses`. + loss_kwargs : dict + Extra keyword arguments forwarded to ``loss_fn``. + + Returns + ------- + torch.Tensor + Scalar loss. + """ + raise NotImplementedError + + def prepare_score_input(self, history, max_len, pad_idx): + """Turn a known history into a padded model input for inference. + + Parameters + ---------- + history : list of int + Known item ids (non-empty). + max_len : int + pad_idx : int + + Returns + ------- + (list of int, int) + ``(input_iids, read_position)`` where ``input_iids`` has length + exactly ``max_len`` (left-padded). Default (causal / next-item): + keep the last ``max_len`` items and read the last position. + """ + hist = list(history)[-max_len:] + input_iids = [pad_idx] * (max_len - len(hist)) + hist + return input_iids, -1 + + def predict_scores(self, model, input_iids, read_position): + """Inference scoring hook. Default: dot-product head via ``predict``.""" + return model.predict(None, input_iids, read_position) diff --git a/cornac/models/transformer_rec/objectives/clm.py b/cornac/models/transformer_rec/objectives/clm.py new file mode 100644 index 00000000..7ef06853 --- /dev/null +++ b/cornac/models/transformer_rec/objectives/clm.py @@ -0,0 +1,95 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import torch + +from .base import Objective + + +class CLMObjective(Objective): + """Causal language modeling: predict every next item from its prefix. + + Requires a causal backbone in the whole-session path. The legacy + prefix path (:meth:`compute_loss_prefix`) reproduces the original + BERT4Rec / GPT2Rec last-position training and is valid for any attention + type (the recommender enforces that). + """ + + VALID_ATTENTION = ("causal",) + uses_mask_token = False + + def __init__(self, pad_idx, mask_idx, rng): + super().__init__(pad_idx, mask_idx, rng) + + def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): + seqs_t = torch.as_tensor(seqs, dtype=torch.long, device=model.dev) + inputs = seqs_t[:, :-1] + targets = seqs_t[:, 1:] + + # Valid loss positions are where the INPUT token is real. With left + # padding this also guarantees a real target and never asks the model + # to predict the first item from an empty prefix. + valid = inputs != self.pad_idx # (B, T-1) + + hidden = model.encode(inputs) # (B, T-1, D) + hidden_flat = hidden[valid] # (M, D) + target_flat = targets[valid] # (M,) + + out_iids = _build_out_iids( + target_flat, sample_negatives, loss_kwargs, model.dev + ) + scores = model.score_positions(hidden_flat, out_iids) + return loss_fn( + scores, + out_iids=out_iids, + batch_size=hidden_flat.size(0), + **loss_kwargs, + ) + + def compute_loss_prefix(self, model, hist_iids, out_iids, loss_fn, loss_kwargs): + """Legacy last-position prefix path (BERT4Rec / GPT2Rec training). + + Parameters + ---------- + model : TransformerRecModel + hist_iids : torch.LongTensor, shape (B, T) + Left-padded prefixes from ``session_seq_iter``. + out_iids : torch.LongTensor, shape (B + N,) + In-batch positives followed by shared negatives. + """ + hidden = model.encode(hist_iids)[:, -1, :] + scores = model.score_positions(hidden, out_iids) + return loss_fn( + scores, + out_iids=out_iids, + batch_size=hist_iids.size(0), + **loss_kwargs, + ) + + +def _build_out_iids(target_flat, sample_negatives, loss_kwargs, device): + """Concatenate positives with sampled negatives. + + The negative count is read from ``loss_kwargs['n_sample']`` (mirroring + the family's ``loss_kwargs = dict(..., n_sample=...)`` convention); + absent or zero yields in-batch negatives only. + """ + n_sample = loss_kwargs.get("n_sample", 0) + if sample_negatives is None or not n_sample: + return target_flat + negatives = torch.as_tensor( + sample_negatives(n_sample), dtype=torch.long, device=device + ) + return torch.cat([target_flat, negatives]) diff --git a/cornac/models/transformer_rec/objectives/mlm.py b/cornac/models/transformer_rec/objectives/mlm.py new file mode 100644 index 00000000..ffb6196f --- /dev/null +++ b/cornac/models/transformer_rec/objectives/mlm.py @@ -0,0 +1,82 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import torch + +from .base import Objective +from .clm import _build_out_iids + + +class MLMObjective(Objective): + """Masked language modeling (BERT4Rec-style Cloze objective). + + Random non-pad positions are replaced by ``mask_idx`` and the model must + recover the original items. Masking always replaces with the mask token + (no 80/10/10 split). Requires a bidirectional backbone. + """ + + VALID_ATTENTION = ("bidirectional",) + uses_mask_token = True + + def __init__(self, pad_idx, mask_idx, rng, mask_prob=0.2): + super().__init__(pad_idx, mask_idx, rng) + self.mask_prob = mask_prob + + def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): + seqs_t = torch.as_tensor(seqs, dtype=torch.long, device=model.dev) + seqs_np = np.asarray(seqs) + B, T = seqs_np.shape + + # Sample mask positions among NON-pad positions; force >= 1 per row. + mask_matrix = np.zeros((B, T), dtype=bool) + for b in range(B): + positions = np.nonzero(seqs_np[b] != self.pad_idx)[0] + if len(positions) == 0: + continue + chosen = positions[self.rng.rand(len(positions)) < self.mask_prob] + if len(chosen) == 0: + chosen = self.rng.choice(positions, size=1) + mask_matrix[b, chosen] = True + + mask_t = torch.as_tensor(mask_matrix, device=model.dev) + inputs = seqs_t.clone() + inputs[mask_t] = self.mask_idx # always replace (no 80/10/10) + + hidden = model.encode(inputs) # (B, T, D) + hidden_flat = hidden[mask_t] # (M, D) + target_flat = seqs_t[mask_t] # (M,) + + out_iids = _build_out_iids( + target_flat, sample_negatives, loss_kwargs, model.dev + ) + scores = model.score_positions(hidden_flat, out_iids) + return loss_fn( + scores, + out_iids=out_iids, + batch_size=hidden_flat.size(0), + **loss_kwargs, + ) + + def prepare_score_input(self, history, max_len, pad_idx): + """Append a mask token after the history and read it. + + Keep the last ``max_len - 1`` items, append ``mask_idx``, and + left-pad to ``max_len``; the mask position (last) is scored. + """ + hist = list(history)[-(max_len - 1):] + input_iids = hist + [self.mask_idx] + input_iids = [pad_idx] * (max_len - len(input_iids)) + input_iids + return input_iids, -1 diff --git a/cornac/models/transformer_rec/objectives/plm.py b/cornac/models/transformer_rec/objectives/plm.py new file mode 100644 index 00000000..98783e43 --- /dev/null +++ b/cornac/models/transformer_rec/objectives/plm.py @@ -0,0 +1,158 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import torch + +from .base import Objective + + +class PLMObjective(Objective): + """Permutation Language Modeling (XLNet two-stream), Transformers4Rec-style. + + Recipe followed + --------------- + Adapted from NVIDIA Merlin Transformers4Rec, + ``transformers4rec/torch/masking.py`` class + :class:`PermutationLanguageModeling` (method + ``_compute_masked_targets_extended``, training branch). The ``perm_mask`` + construction is reproduced verbatim: a random factorization order is drawn + per row, non-target positions are pinned to ``-1`` (so every query may + attend to them and they never leak into a masked target), and + + ``perm_mask[b, q, k] = (perm_index[q] <= perm_index[k]) & mask_labels[k]`` + + which forbids query ``q`` from attending to a masked key ``k`` unless ``k`` + precedes ``q`` in the factorization order. Because ``perm_index[q] == + perm_index[k]`` for ``q == k``, a target can never attend to itself, so the + original item ids are fed as inputs (no ``mask_idx`` replacement during + training) and XLNet's query (``g``) stream produces each prediction. + + Deviations from Transformers4Rec + -------------------------------- + * Target selection uses an i.i.d. Bernoulli(``mask_prob``) draw over the + non-pad positions (as in MLM), not T4R's span-based sampling. This keeps + the objective consistent with the other TransformerRec objectives and is + the behaviour requested for this integration. + * ``target_mapping`` is built in the compact ``(B, K, T)`` form (one row per + selected target, ``K`` = per-batch max target count, short rows zero- + padded and tracked by a validity mask) so XLNet returns ``(B, K, D)`` + directly. T4R instead uses a full ``(B, T, T)`` identity and relies on the + padded labels to drop non-targets; the compact form is equivalent but + avoids scoring the non-target positions. + + Parameters + ---------- + pad_idx: int + Padding item index (``item_num``). + mask_idx: int + Mask item index (``item_num + 1``), used only at inference time. + rng: numpy.random.RandomState + Random state used for target selection and factorization order. + mask_prob: float, optional, default: 0.2 + Per-position probability of selecting a non-pad item as a prediction + target. At least one target is kept per row. + """ + + VALID_ATTENTION = ("bidirectional",) + uses_mask_token = True + + def __init__(self, pad_idx, mask_idx, rng, mask_prob=0.2): + super().__init__(pad_idx, mask_idx, rng) + self.mask_prob = mask_prob + + def _select_targets(self, non_pad): + """Bernoulli target selection over non-pad positions. + + Guarantees at least one target and at least one visible (non-target) + item per row, following the T4R MLM safeguards. + """ + B, T = non_pad.shape + mask_labels = (self.rng.random((B, T)) < self.mask_prob) & non_pad + for b in range(B): + valid = np.where(non_pad[b])[0] + if len(valid) == 0: + continue + if not mask_labels[b].any(): + mask_labels[b, self.rng.choice(valid)] = True + # If every non-pad item is a target, unmask one to keep context. + if len(valid) > 1 and mask_labels[b].sum() == len(valid): + masked = np.where(mask_labels[b])[0] + mask_labels[b, self.rng.choice(masked)] = False + return mask_labels + + def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): + device = model.dev + B, T = seqs.shape + non_pad = seqs != self.pad_idx + mask_labels = self._select_targets(non_pad) + + # Random factorization-order attention mask (T4R recipe). + perm_mask = np.zeros((B, T, T), dtype=np.float32) + for b in range(B): + perm_index = self.rng.permutation(T).astype(np.int64) + perm_index[~mask_labels[b]] = -1 + perm_mask[b] = ( + perm_index[:, None] <= perm_index[None, :] + ) & mask_labels[b][None, :] + + # Compact target mapping (B, K, T) with a validity mask. + counts = mask_labels.sum(axis=1) + K = int(counts.max()) + target_mapping = np.zeros((B, K, T), dtype=np.float32) + target_items = np.zeros((B, K), dtype=np.int64) + validity = np.zeros((B, K), dtype=bool) + for b in range(B): + positions = np.where(mask_labels[b])[0] + for j, p in enumerate(positions): + target_mapping[b, j, p] = 1.0 + target_items[b, j] = seqs[b, p] + validity[b, j] = True + + inputs = torch.as_tensor(seqs, dtype=torch.long, device=device) + perm_t = torch.as_tensor(perm_mask, device=device) + tm_t = torch.as_tensor(target_mapping, device=device) + + # g-stream outputs, one per target slot: (B, K, D). + g = model.encode(inputs, perm_mask=perm_t, target_mapping=tm_t) + + valid_t = torch.as_tensor(validity, device=device) + hidden = g[valid_t] # (M, D) + targets = torch.as_tensor(target_items, dtype=torch.long, device=device)[ + valid_t + ] # (M,) + M = targets.shape[0] + + out_iids = targets + n_neg = loss_kwargs.get("n_sample", 0) or 0 + if sample_negatives is not None and n_neg > 0: + negs = torch.as_tensor( + sample_negatives(n_neg), dtype=torch.long, device=device + ) + out_iids = torch.cat([targets, negs]) + + scores = model.score_positions(hidden, out_iids) # (M, M+N) + return loss_fn(scores, out_iids=out_iids, batch_size=M, **loss_kwargs) + + def prepare_score_input(self, history, max_len, pad_idx): + """Mask-append serving (plain bidirectional pass, as T4R does). + + Truncate to the last ``max_len - 1`` items, append ``mask_idx``, then + left-pad to ``max_len``. The final (mask) position is read. + """ + hist = list(history)[-(max_len - 1):] + seq = hist + [self.mask_idx] + seq = [pad_idx] * (max_len - len(seq)) + seq + return seq, -1 diff --git a/cornac/models/transformer_rec/objectives/rtd.py b/cornac/models/transformer_rec/objectives/rtd.py new file mode 100644 index 00000000..d3546c1e --- /dev/null +++ b/cornac/models/transformer_rec/objectives/rtd.py @@ -0,0 +1,171 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .base import Objective +from .clm import _build_out_iids + + +class RTDObjective(Objective): + """Replacement Token Detection (ELECTRA), tied-generator variant. + + Recipe followed + --------------- + Adapted from NVIDIA Merlin Transformers4Rec, + ``transformers4rec/torch/masking.py`` class + :class:`ReplacementLanguageModeling` (which subclasses + :class:`MaskedLanguageModeling`) with its ``get_fake_tokens`` / + ``sample_from_softmax`` corruption step, in the **tied-generator** + configuration (T4R's ``rtd_tied_generator``): the main transformer body + plays both the generator and the discriminator roles. Per batch: + + 1. MLM-mask the batch (Bernoulli ``mask_prob`` over non-pad positions, at + least one masked and at least one visible per row). + 2. Generator pass: the main model encodes the masked sequence; masked- + position hidden states are scored against the shared ``item_emb`` under + the same ``(M, M+N)`` diagonal-positive contract used by every other + objective, giving the MLM (generator) loss. + 3. Replacements are sampled from the model's own softmax over the real + items (``torch.multinomial``, detached so no gradient flows through + sampling) and scattered into the masked positions to build a corrupted + sequence (T4R ``get_fake_tokens``). Positions whose sample equals the + original count as "original" (ELECTRA convention). + 4. Discriminator pass: the same body encodes the corrupted sequence; a + per-position ``Linear(D, 1)`` head classifies original-vs-replaced over + every non-pad position with BCE-with-logits. + 5. ``total = mlm_loss + rtd_lambda * disc_loss``. + + Serving head + ------------ + Standard MLM serving: append ``mask_idx`` to the history and read its + position through the main body (inherited default ``predict_scores``). + Because the same body is trained on masked inputs in step (2), the + serving path is fully trained — and independent of ``disc_head``, so + ``model_selection='best'`` snapshots of the main model alone cover it. + + Deviations from ELECTRA / Transformers4Rec + ------------------------------------------ + * ELECTRA's separate *small* generator (T4R's untied variant with + ``generator_size_ratio``) is not implemented: with zero-shot serving + there is no fine-tuning step to exploit a discriminator-only body, so + the untrained-serving-head problem makes the untied variant unusable + here (empirically test AUC < 0.5). + * Replacements are sampled with plain ``torch.multinomial`` on the softmax + over the real items only (pad/mask columns excluded), rather than T4R's + Gumbel-argmax; both are unbiased categorical draws. + + Parameters + ---------- + pad_idx: int + Padding item index (``item_num``). + mask_idx: int + Mask item index (``item_num + 1``). + rng: numpy.random.RandomState + Random state for masked-position selection. + mask_prob: float, optional, default: 0.2 + Per-position masking probability. At least one item is masked per row. + rtd_lambda: float, optional, default: 1.0 + Weight of the discriminator (RTD) loss relative to the MLM loss. + ``rtd_lambda -> 0`` recovers plain MLM. ELECTRA's original 50 is tuned + for fine-tuning pipelines; for zero-shot ranking the discriminator + term competes with the item-embedding alignment, and Diginetica + sweeps show ranking quality degrading monotonically with lambda — + tune downward if ranking metrics matter most. + """ + + VALID_ATTENTION = ("bidirectional",) + uses_mask_token = True + + def __init__(self, pad_idx, mask_idx, rng, mask_prob=0.2, rtd_lambda=1.0): + super().__init__(pad_idx, mask_idx, rng) + self.mask_prob = mask_prob + self.rtd_lambda = rtd_lambda + self.disc_head = None + + def build(self, model, device): + self.disc_head = nn.Linear(model.item_emb.embedding_dim, 1) + self.disc_head.to(device) + + def parameters(self): + return list(self.disc_head.parameters()) + + def _mlm_mask(self, non_pad): + """Bernoulli MLM masking with T4R's at-least-one / not-all safeguards.""" + B, T = non_pad.shape + mask_labels = (self.rng.random((B, T)) < self.mask_prob) & non_pad + for b in range(B): + valid = np.where(non_pad[b])[0] + if len(valid) == 0: + continue + if not mask_labels[b].any(): + mask_labels[b, self.rng.choice(valid)] = True + if len(valid) > 1 and mask_labels[b].sum() == len(valid): + masked = np.where(mask_labels[b])[0] + mask_labels[b, self.rng.choice(masked)] = False + return mask_labels + + def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): + device = model.dev + non_pad = seqs != self.pad_idx + mask_labels = self._mlm_mask(non_pad) + + seqs_t = torch.as_tensor(seqs, dtype=torch.long, device=device) + non_pad_t = torch.as_tensor(non_pad, device=device) + mask_t = torch.as_tensor(mask_labels, device=device) # (B, T) bool + + # (1-2) Generator pass: MLM through the main body. + masked_ids = seqs_t.clone() + masked_ids[mask_t] = self.mask_idx + hidden = model.encode(masked_ids) # (B, T, D) + mlm_hidden = hidden[mask_t] # (M, D) + targets = seqs_t[mask_t] # (M,) + + out_iids = _build_out_iids(targets, sample_negatives, loss_kwargs, device) + mlm_scores = model.score_positions(mlm_hidden, out_iids) # (M, M+N) + mlm_loss = loss_fn( + mlm_scores, out_iids=out_iids, batch_size=targets.shape[0], **loss_kwargs + ) + + # (3) Sample replacements from the model's own predictions (detached). + with torch.no_grad(): + real_ids = torch.arange(model.item_num, device=device) + real_scores = model.score_positions(mlm_hidden, real_ids) # (M, item_num) + probs = torch.softmax(real_scores, dim=-1) + sampled = torch.multinomial(probs, num_samples=1).squeeze(-1) # (M,) + + corrupted = seqs_t.clone() + corrupted[mask_t] = sampled # same row-major order as mlm_hidden / targets + + # (4) Discriminator pass: same body classifies original vs replaced. + disc_hidden = model.encode(corrupted) # (B, T, D) + disc_logits = self.disc_head(disc_hidden).squeeze(-1) # (B, T) + disc_labels = (corrupted != seqs_t).float() # 1 where replaced + disc_loss = F.binary_cross_entropy_with_logits( + disc_logits[non_pad_t], disc_labels[non_pad_t] + ) + + # (5) Joint objective. + return mlm_loss + self.rtd_lambda * disc_loss + + def prepare_score_input(self, history, max_len, pad_idx): + """Mask-append serving through the main body (trained by the MLM pass).""" + hist = list(history)[-(max_len - 1):] + seq = hist + [self.mask_idx] + seq = [pad_idx] * (max_len - len(seq)) + seq + return seq, -1 diff --git a/cornac/models/transformer_rec/recom_transformer_rec.py b/cornac/models/transformer_rec/recom_transformer_rec.py new file mode 100644 index 00000000..032e44e6 --- /dev/null +++ b/cornac/models/transformer_rec/recom_transformer_rec.py @@ -0,0 +1,493 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +from tqdm.auto import trange + +from cornac.models.recommender import NextItemRecommender + +from ...utils import get_rng +from ..seq_utils import val_score +from .backbones import ATTENTION_TYPES + +SUPPORTED_LOSSES = ( + "bce", + "ce", + "bpr", + "bpr-max", + "softmax", + "cross-entropy", + "xe_softmax", + "top1", +) + +VALID_OBJECTIVES = ("clm", "mlm", "plm", "rtd") +VALID_LOSS_AT = ("all", "last") + + +class TransformerRec(NextItemRecommender): + """TransformerRec: unified transformer next-item recommender. + + A single item-embedding scoring head over a swappable HuggingFace + transformer backbone (``bert``/``gpt2``/``xlnet``/``electra``), trained + with one of four self-supervised objectives (``clm``/``mlm``/``plm``/ + ``rtd``). This subsumes BERT4Rec and the Transformers4Rec family in one + model; see the validity matrix below for the objective/backbone + combinations that are supported. + + Validity matrix (enforced at construction time) + ----------------------------------------------- + ============================== ============================ ========== + objective valid backbones loss_at + ============================== ============================ ========== + clm (loss_at='all') gpt2 (causal only) 'all' + clm (loss_at='last', legacy) any backbone 'last' + mlm bert, electra, xlnet 'all' + plm xlnet only 'all' + rtd bert, electra 'all' + ============================== ============================ ========== + + Parameters + ---------- + name: str, default: 'TransformerRec' + The name of the recommender model. + + backbone: str, default: 'bert' + Transformer backbone. One of 'bert', 'gpt2', 'xlnet', 'electra'. + + objective: str, default: 'mlm' + Self-supervised training objective. One of 'clm' (causal LM), + 'mlm' (masked LM / Cloze), 'plm' (permutation LM), 'rtd' + (replaced-token detection). + + loss_at: str, default: 'all' + Where the training loss is applied. 'all' scores every eligible + position of the session; 'last' (only valid with ``objective='clm'``) + is the legacy Cornac prefix-breakdown setting that scores the last + position of each expanded prefix and works with any backbone. + + embedding_dim: int, optional, default: 100 + Item embedding / hidden dimension. + + loss: str, optional, default: 'ce' + Loss function. Supported: 'bce', 'ce', 'bpr', 'bpr-max', 'softmax' + (a.k.a 'cross-entropy' / 'xe_softmax'), 'top1'. + + batch_size: int, optional, default: 512 + + learning_rate: float, optional, default: 0.001 + + n_sample: int, optional, default: 2048 + Number of negative samples shared per mini-batch (0 disables extra + sampled negatives, using in-batch negatives only). + + sample_alpha: float, optional, default: 0.5 + Popularity-based negative sampling exponent. + + n_epochs: int, optional, default: 10 + + max_len: int, optional, default: 50 + Maximum session length fed to the encoder. + + num_blocks: int, optional, default: 2 + Number of transformer layers. + + num_heads: int, optional, default: 1 + Number of attention heads. + + dropout: float, optional, default: 0.2 + + l2_reg: float, optional, default: 0.0 + + bpreg: float, optional, default: 1.0 + Regularization coefficient for the 'bpr-max' loss. + + elu_param: float, optional, default: 0.5 + ELU parameter for the 'bpr-max' loss. + + mask_prob: float, optional, default: 0.2 + Per-position masking probability for the 'mlm', 'plm', and 'rtd' + objectives. + + rtd_lambda: float, optional, default: 1.0 + (RTD only) weight of the discriminator loss relative to the MLM + (generator) loss. Larger values favor the discriminative signal at + the expense of ranking alignment; ``0`` recovers plain MLM. + + device: str, optional, default: 'cpu' + Set to 'cuda' for GPU support. + + trainable: bool, optional, default: True + When False, the model will not be re-trained. + + verbose: bool, optional, default: False + When True, running logs are displayed. + + seed: int, optional, default: None + Random seed for weight initialization and negative sampling. + + model_selection: str, optional, default: 'last' + One of 'last' or 'best'. When 'best', the model with the highest + validation score (evaluated every ``val_eval_every`` epochs) is + restored at the end of ``fit``. + + val_eval_every: int, optional, default: 5 + val_k: int, optional, default: 20 + val_metric: str, optional, default: 'recall' + Cutoff and metric used for best-on-val selection. See + :func:`cornac.models.seq_utils.val_score`. + + Note + ---- + * ``backbone='bert', objective='mlm'`` reproduces the canonical BERT4Rec + Cloze training (Sun et al., 2019). + * ``backbone='gpt2', objective='clm'`` reproduces the Transformers4Rec + GPT-2 / causal-LM setup (Moreira et al., 2021). + * ``objective='clm', loss_at='last'`` is the legacy Cornac prefix + breakdown (next-item-at-last-position), valid with any backbone. + * With ``model_selection='best'`` only the main model's ``state_dict`` is + snapshotted/restored; RTD's auxiliary discriminator head is not + checkpoint-restored since it is not used at serving. + + References + ---------- + Moreira, G. de S. P., Rabhi, S., Lee, J. M., Ak, R., & Oldridge, E. + (2021). Transformers4Rec: Bridging the gap between NLP and sequential / + session-based recommendation. RecSys. + + Sun, F., Liu, J., Wu, J., Pei, C., Lin, X., Ou, W., & Jiang, P. (2019). + BERT4Rec: Sequential recommendation with bidirectional encoder + representations from transformer. CIKM. + + Clark, K., Luong, M.-T., Le, Q. V., & Manning, C. D. (2020). ELECTRA: + Pre-training text encoders as discriminators rather than generators. + ICLR. + + Yang, Z., Dai, Z., Yang, Y., Carbonell, J., Salakhutdinov, R., & Le, + Q. V. (2019). XLNet: Generalized autoregressive pretraining for language + understanding. NeurIPS. + """ + + def __init__( + self, + name="TransformerRec", + backbone="bert", + objective="mlm", + loss_at="all", + embedding_dim=100, + loss="ce", + batch_size=512, + learning_rate=0.001, + n_sample=2048, + sample_alpha=0.5, + n_epochs=10, + max_len=50, + num_blocks=2, + num_heads=1, + dropout=0.2, + l2_reg=0.0, + bpreg=1.0, + elu_param=0.5, + mask_prob=0.2, + rtd_lambda=1.0, + device="cpu", + trainable=True, + verbose=False, + seed=None, + model_selection="last", + val_eval_every=5, + val_k=20, + val_metric="recall", + ): + super().__init__(name, trainable=trainable, verbose=verbose) + if objective not in VALID_OBJECTIVES: + raise ValueError( + f"objective='{objective}' not supported; choose from {VALID_OBJECTIVES}" + ) + if loss_at not in VALID_LOSS_AT: + raise ValueError( + f"loss_at='{loss_at}' not supported; choose from {VALID_LOSS_AT}" + ) + if backbone not in ATTENTION_TYPES: + raise ValueError( + f"Unknown backbone '{backbone}'; choose from {sorted(ATTENTION_TYPES)}" + ) + if loss not in SUPPORTED_LOSSES: + raise ValueError( + f"loss='{loss}' not supported; choose from {SUPPORTED_LOSSES}" + ) + if model_selection not in ("last", "best"): + raise ValueError( + f"model_selection='{model_selection}' not supported; choose 'last' or 'best'" + ) + + self._validate_combo(objective, loss_at, backbone) + + self.backbone = backbone + self.objective = objective + self.loss_at = loss_at + self.embedding_dim = embedding_dim + self.loss = loss + self.batch_size = batch_size + self.learning_rate = learning_rate + self.n_sample = n_sample + self.sample_alpha = sample_alpha + self.n_epochs = n_epochs + self.max_len = max_len + self.num_blocks = num_blocks + self.num_heads = num_heads + self.dropout = dropout + self.l2_reg = l2_reg + self.bpreg = bpreg + self.elu_param = elu_param + self.mask_prob = mask_prob + self.rtd_lambda = rtd_lambda + self.device = device + self.seed = seed + self.rng = get_rng(seed) + self.model_selection = model_selection + self.val_eval_every = val_eval_every + self.val_k = val_k + self.val_metric = val_metric + + @staticmethod + def _validate_combo(objective, loss_at, backbone): + """Enforce the objective/backbone/loss_at validity matrix.""" + attn = ATTENTION_TYPES[backbone] + if objective == "clm": + if loss_at == "all" and attn != "causal": + raise ValueError( + f"objective='clm' with loss_at='all' requires a causal " + f"backbone (e.g. 'gpt2'), but backbone='{backbone}' is " + f"'{attn}'. Use loss_at='last' for the legacy prefix mode " + f"with this backbone." + ) + return # loss_at='last' works with any backbone + + # mlm / plm / rtd are all whole-session ('all') objectives. + if loss_at != "all": + raise ValueError( + f"objective='{objective}' only supports loss_at='all', " + f"got loss_at='{loss_at}'." + ) + if objective == "mlm": + if attn != "bidirectional": + raise ValueError( + f"objective='mlm' requires a bidirectional backbone " + f"(bert, electra, xlnet), but backbone='{backbone}' is " + f"'{attn}'." + ) + elif objective == "plm": + if backbone != "xlnet": + raise ValueError( + f"objective='plm' requires backbone='xlnet', " + f"got backbone='{backbone}'." + ) + elif objective == "rtd": + if backbone not in ("bert", "electra"): + raise ValueError( + f"objective='rtd' requires backbone in ('bert', 'electra'), " + f"got backbone='{backbone}'." + ) + + def _build_objective(self): + """Instantiate the objective implementation from its name.""" + from .objectives import ( + CLMObjective, + MLMObjective, + PLMObjective, + RTDObjective, + ) + + name = self.objective + if name == "clm": + return CLMObjective(self.pad_idx, self.mask_idx, self.rng) + if name == "mlm": + return MLMObjective( + self.pad_idx, self.mask_idx, self.rng, mask_prob=self.mask_prob + ) + if name == "plm": + return PLMObjective( + self.pad_idx, self.mask_idx, self.rng, mask_prob=self.mask_prob + ) + return RTDObjective( + self.pad_idx, + self.mask_idx, + self.rng, + mask_prob=self.mask_prob, + rtd_lambda=self.rtd_lambda, + ) + + def fit(self, train_set, val_set=None): + super().fit(train_set, val_set) + if not self.trainable: + return self + + import torch + + from .transformer_rec import TransformerRecModel + from ..seq_utils import ( + build_neg_sampler, + padded_session_iter, + session_seq_iter, + ) + from ..seq_utils.losses import get_loss_function + + torch.manual_seed(self.seed if self.seed is not None else 0) + + use_prefix = self.objective == "clm" and self.loss_at == "last" + + self.pad_idx = self.total_items + self.mask_idx = self.total_items + 1 + self.model = TransformerRecModel( + item_num=self.total_items, + backbone=self.backbone, + embedding_dim=self.embedding_dim, + maxlen=self.max_len, + n_layers=self.num_blocks, + n_heads=self.num_heads, + dropout=self.dropout, + device=self.device, + ) + + # The built instance lives in ``objective_`` so ``self.objective`` + # stays the plain string hyperparameter (kept intact for clone()). + objective = self._build_objective() + objective.build(self.model, self.device) + self.objective_ = objective + + loss_fn = get_loss_function(self.loss) + loss_kwargs = dict( + bpreg=self.bpreg, elu_param=self.elu_param, n_sample=self.n_sample + ) + + if self.n_sample > 0: + item_indices, item_dist = build_neg_sampler( + train_set.uir_tuple, self.sample_alpha + ) + sample_negatives = lambda n: self.rng.choice( + item_indices, size=n, replace=True, p=item_dist + ) + else: + sample_negatives = None + + opt = torch.optim.Adam( + list(self.model.parameters()) + list(objective.parameters()), + lr=self.learning_rate, + betas=(0.9, 0.98), + ) + + best_val = -float("inf") + best_state = None + progress_bar = trange(1, self.n_epochs + 1, disable=not self.verbose) + for epoch_id in progress_bar: + self.model.train() + total_loss = 0.0 + cnt = 0 + + if use_prefix: + for inc, (in_uids, hist_iids, out_iids) in enumerate( + session_seq_iter( + self.train_set, + pad_index=self.pad_idx, + batch_size=self.batch_size, + max_len=self.max_len, + n_sample=self.n_sample, + sample_alpha=self.sample_alpha, + rng=self.rng, + shuffle=True, + ) + ): + if len(hist_iids) < 2: + continue + hist_iids_t = torch.tensor( + hist_iids, dtype=torch.long, device=self.device + ) + out_iids_t = torch.tensor( + out_iids, dtype=torch.long, device=self.device + ) + + self.model.zero_grad() + L = objective.compute_loss_prefix( + self.model, hist_iids_t, out_iids_t, loss_fn, loss_kwargs + ) + if self.l2_reg > 0: + for p in self.model.parameters(): + L = L + self.l2_reg * torch.norm(p) + + L.backward() + opt.step() + + total_loss += L.cpu().detach().numpy() * len(hist_iids) + cnt += len(hist_iids) + if inc % 10 == 0 and cnt > 0: + progress_bar.set_postfix(loss=(total_loss / cnt)) + else: + for inc, (uids, seqs) in enumerate( + padded_session_iter( + self.train_set, + pad_index=self.pad_idx, + batch_size=self.batch_size, + max_len=self.max_len, + rng=self.rng, + shuffle=True, + ) + ): + if len(seqs) < 2: + continue + + self.model.zero_grad() + L = objective.compute_loss( + self.model, seqs, sample_negatives, loss_fn, loss_kwargs + ) + if self.l2_reg > 0: + for p in self.model.parameters(): + L = L + self.l2_reg * torch.norm(p) + + L.backward() + opt.step() + + total_loss += L.cpu().detach().numpy() * len(seqs) + cnt += len(seqs) + if inc % 10 == 0 and cnt > 0: + progress_bar.set_postfix(loss=(total_loss / cnt)) + + if ( + self.model_selection == "best" + and val_set is not None + and epoch_id % self.val_eval_every == 0 + ): + score = val_score( + self, self.train_set, val_set, metric=self.val_metric, k=self.val_k + ) + if score is not None and score > best_val: + best_val = score + best_state = { + n: p.detach().clone() + for n, p in self.model.state_dict().items() + } + + if self.model_selection == "best" and best_state is not None: + self.model.load_state_dict(best_state) + return self + + def score(self, user_idx, history_items, **kwargs): + if len(history_items) == 0: + return np.ones(self.total_items, dtype="float") + inp, pos = self.objective_.prepare_score_input( + list(history_items), self.max_len, self.pad_idx + ) + self.model.eval() + return self.objective_.predict_scores(self.model, inp, pos) diff --git a/cornac/models/transformer_rec/requirements.txt b/cornac/models/transformer_rec/requirements.txt new file mode 100644 index 00000000..c50f9e17 --- /dev/null +++ b/cornac/models/transformer_rec/requirements.txt @@ -0,0 +1,2 @@ +torch>=1.12.0 +transformers diff --git a/cornac/models/transformer_rec/transformer_rec.py b/cornac/models/transformer_rec/transformer_rec.py new file mode 100644 index 00000000..bb7d12f6 --- /dev/null +++ b/cornac/models/transformer_rec/transformer_rec.py @@ -0,0 +1,183 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import torch +import torch.nn as nn + +from .backbones import get_backbone + + +class TransformerRecModel(nn.Module): + """Unified HuggingFace-backbone encoder for next-item recommendation. + + A single item-embedding table feeds a swappable transformer backbone + (``bert``/``gpt2``/``xlnet``/``electra``); candidate items are scored by + the dot product of a position's hidden state with the item embeddings + plus per-item biases. Training objectives (CLM/MLM/PLM/RTD) drive the + model through :meth:`encode` and :meth:`score_positions`. + + Vocabulary layout (frozen): ``pad_idx = item_num`` and + ``mask_idx = item_num + 1``. All embeddings have ``item_num + 2`` rows + regardless of objective; only the first ``item_num`` rows are real items. + + Parameters + ---------- + item_num : int + Number of real items. Ids are integers in ``[0, item_num)``. + backbone : str, default 'bert' + Backbone name registered in :mod:`.backbones`. + embedding_dim : int, default 100 + Item/hidden embedding dimension. + maxlen : int, default 20 + Maximum sequence length fed to the encoder. + n_layers : int, default 2 + Number of transformer layers. + n_heads : int, default 1 + Number of attention heads. + dropout : float, default 0.1 + Dropout probability inside the backbone. + init_std : float, default 0.02 + Standard deviation for item-embedding initialization. + device : str, default 'cpu' + Torch device. + """ + + def __init__( + self, + item_num, + backbone="bert", + embedding_dim=100, + maxlen=20, + n_layers=2, + n_heads=1, + dropout=0.1, + init_std=0.02, + device="cpu", + ): + super().__init__() + self.item_num = item_num + self.pad_idx = item_num + self.mask_idx = item_num + 1 + self.maxlen = maxlen + self.dev = device + self.init_std = init_std + + build_fn, attention_type = get_backbone(backbone) + self.attention_type = attention_type + + vocab_size = item_num + 2 + self.item_emb = nn.Embedding( + num_embeddings=vocab_size, + embedding_dim=embedding_dim, + padding_idx=self.pad_idx, + ) + self.backbone = build_fn( + vocab_size=vocab_size, + embedding_dim=embedding_dim, + max_len=maxlen, + n_layers=n_layers, + n_heads=n_heads, + dropout=dropout, + pad_idx=self.pad_idx, + ) + self.item_biases = nn.Embedding(vocab_size, 1, padding_idx=self.pad_idx) + + self._init_weights() + self.to(device) + + def _init_weights(self): + self.item_emb.weight.data.normal_(mean=0.0, std=self.init_std) + self.item_emb.weight.data[self.pad_idx].zero_() + self.item_biases.weight.data.zero_() + + def encode(self, hist_iids, **backbone_kwargs): + """Encode a batch of item-id sequences. + + Parameters + ---------- + hist_iids : torch.LongTensor, shape (B, T) + Left-padded item ids (``pad_idx`` in padding slots). The mask + token (``mask_idx``) counts as a real token in the attention mask + because it is ``!= pad_idx``. + **backbone_kwargs + Extra keyword arguments forwarded to the backbone forward call + (e.g. ``perm_mask`` / ``target_mapping`` for XLNet). + + Returns + ------- + torch.Tensor + The backbone ``last_hidden_state``. Shape ``(B, T, D)`` for the + plain call, or whatever the backbone returns for special kwargs + (e.g. ``(B, K, D)`` for XLNet with ``target_mapping``). + """ + attention_mask = (hist_iids != self.pad_idx).long() + embeds = self.item_emb(hist_iids) + out = self.backbone( + inputs_embeds=embeds, attention_mask=attention_mask, **backbone_kwargs + ) + return out.last_hidden_state + + def score_positions(self, hidden, out_iids): + """Score candidate items at each hidden position. + + Parameters + ---------- + hidden : torch.Tensor, shape (M, D) + Hidden states gathered at the loss positions. + out_iids : torch.LongTensor, shape (M + N,) + Candidate item ids: the ``M`` positives followed by ``N`` shared + negatives. + + Returns + ------- + torch.Tensor, shape (M, M + N) + Score matrix (positives on the diagonal). + """ + return torch.mm(hidden, self.item_emb(out_iids).T) + self.item_biases(out_iids).T + + @torch.no_grad() + def predict(self, user_ids, log_seqs, read_position=-1, item_indices=None): + """Score all real items for a single sequence. + + Parameters + ---------- + user_ids : ignored + Present for signature compatibility with the model family. + log_seqs : array-like or torch.LongTensor, shape (1, T) or (T,) + A single left-padded item-id sequence. + read_position : int, default -1 + Position whose hidden state is used for scoring. + item_indices : array-like, optional + Candidate items to score. Defaults to all real items + ``arange(item_num)`` (pad and mask rows are excluded). + + Returns + ------- + numpy.ndarray, shape (len(item_indices),) + Scores for the requested items. + """ + if item_indices is None: + item_indices = torch.arange(self.item_num, device=self.dev) + else: + item_indices = torch.as_tensor( + item_indices, dtype=torch.long, device=self.dev + ) + if not isinstance(log_seqs, torch.Tensor): + log_seqs = torch.as_tensor(log_seqs, dtype=torch.long, device=self.dev) + if log_seqs.dim() == 1: + log_seqs = log_seqs.unsqueeze(0) + hidden = self.encode(log_seqs)[:, read_position, :] + scores = self.score_positions(hidden, item_indices) + return scores.squeeze().detach().cpu().numpy() diff --git a/examples/transformer_rec_diginetica.py b/examples/transformer_rec_diginetica.py index 28564f6f..b1398528 100644 --- a/examples/transformer_rec_diginetica.py +++ b/examples/transformer_rec_diginetica.py @@ -14,17 +14,19 @@ # ============================================================================ """Transformer-based next-item recommenders on Diginetica. -SASRec, BERT4Rec, and GPT2Rec share one scoring head (encode the current -session, take the last-position hidden state, dot-product against item -embeddings) and differ only in the sequence encoder: +TransformerRec is one model with two axes of configuration: a HuggingFace +backbone (bert/gpt2/xlnet/electra) and a language-modeling objective +(clm/mlm/plm/rtd). This example compares: -- SASRec : its own causal self-attention stack (torch only) -- BERT4Rec : a HuggingFace BERT encoder -- GPT2Rec : a HuggingFace GPT-2 decoder +- SASRec : causal self-attention baseline (torch only) +- TransformerRec gpt2+clm (all) : causal LM, loss at every position +- TransformerRec gpt2+clm (last) : legacy prefix breakdown, loss at last + position only (the old GPT2Rec behavior) +- TransformerRec bert+mlm : BERT4Rec-style Cloze training -BERT4Rec and GPT2Rec require the ``transformers`` package (see each model's -requirements.txt). All three use the next-item-at-last-position objective, not -the canonical MLM/CLM losses in Transformers4Rec paper. +TransformerRec requires the ``transformers`` package (see the model's +requirements.txt). The clm-all vs clm-last pair isolates the effect of the +training setting under identical architecture and hyperparameters. """ import torch @@ -33,7 +35,7 @@ from cornac.datasets import diginetica from cornac.eval_methods import NextItemEvaluation from cornac.metrics import MRR, NDCG, Recall -from cornac.models import BERT4Rec, GPT2Rec, GRU4Rec, SASRec +from cornac.models import SASRec, TransformerRec DEVICE = "cuda:0" if torch.cuda.is_available() else "cpu" print(f"using device: {DEVICE}") @@ -52,7 +54,7 @@ fmt="USIT", ) -transformer = dict( +shared = dict( embedding_dim=64, loss="cross-entropy", n_sample=512, @@ -71,26 +73,30 @@ ) models = [ - GRU4Rec( - layers=[100], - loss="cross-entropy", - dropout_p_hidden=0.3, - sample_alpha=0.75, - n_sample=512, - batch_size=64, - learning_rate=0.1, - n_epochs=50, - model_selection="best", - val_eval_every=5, - val_metric="recall", - val_k=20, - device=DEVICE, - verbose=True, - seed=123, + SASRec(learning_rate=0.01, **shared), + TransformerRec( + name="TransformerRec-gpt2-clm-all", + backbone="gpt2", + objective="clm", + loss_at="all", + learning_rate=0.001, + **shared, + ), + TransformerRec( + name="TransformerRec-gpt2-clm-last", + backbone="gpt2", + objective="clm", + loss_at="last", + learning_rate=0.001, + **shared, + ), + TransformerRec( + name="TransformerRec-bert-mlm", + backbone="bert", + objective="mlm", + learning_rate=0.01, + **shared, ), - SASRec(learning_rate=0.01, **transformer), - BERT4Rec(learning_rate=0.01, **transformer), - GPT2Rec(learning_rate=0.001, **transformer), ] metrics = [ From 66cb4ecc5f740b371ea40695a32eecc0a72caa5d Mon Sep 17 00:00:00 2001 From: hieuddo Date: Sun, 5 Jul 2026 20:12:11 +0800 Subject: [PATCH 2/5] cleanup: deprecated standalone BERT4Rec and GPT2Rec --- cornac/models/bert4rec/__init__.py | 16 -- cornac/models/bert4rec/bert4rec.py | 115 ------------ cornac/models/bert4rec/recom_bert4rec.py | 227 ----------------------- cornac/models/bert4rec/requirements.txt | 2 - cornac/models/gpt2rec/__init__.py | 16 -- cornac/models/gpt2rec/gpt2rec.py | 110 ----------- cornac/models/gpt2rec/recom_gpt2rec.py | 227 ----------------------- cornac/models/gpt2rec/requirements.txt | 2 - 8 files changed, 715 deletions(-) delete mode 100644 cornac/models/bert4rec/__init__.py delete mode 100644 cornac/models/bert4rec/bert4rec.py delete mode 100644 cornac/models/bert4rec/recom_bert4rec.py delete mode 100644 cornac/models/bert4rec/requirements.txt delete mode 100644 cornac/models/gpt2rec/__init__.py delete mode 100644 cornac/models/gpt2rec/gpt2rec.py delete mode 100644 cornac/models/gpt2rec/recom_gpt2rec.py delete mode 100644 cornac/models/gpt2rec/requirements.txt diff --git a/cornac/models/bert4rec/__init__.py b/cornac/models/bert4rec/__init__.py deleted file mode 100644 index af46ce60..00000000 --- a/cornac/models/bert4rec/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -from .recom_bert4rec import BERT4Rec diff --git a/cornac/models/bert4rec/bert4rec.py b/cornac/models/bert4rec/bert4rec.py deleted file mode 100644 index 3213059b..00000000 --- a/cornac/models/bert4rec/bert4rec.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import torch -import torch.nn as nn - - -class BERT4RecModel(nn.Module): - """BERT4Rec: bidirectional transformer encoder for sequence rec. - - Uses HuggingFace's :class:`~transformers.BertModel` as the backbone. The - sequence-final hidden state is used to score candidate items via the dot - product with their output embeddings (separate "head" linear or tied to - ``item_emb``). - - Returns ``(B, B+N)`` score matrices when called as - ``forward(_, hist_iids, out_iids, return_hidden=False)``. - """ - - def __init__( - self, - item_num, - embedding_dim=100, - maxlen=20, - n_layers=2, - n_heads=1, - dropout=0.1, - pad_idx=-1, - init_std=0.02, - device="cpu", - ): - super().__init__() - from transformers.models.bert import BertConfig, BertModel - - self.item_num = item_num - self.pad_idx = pad_idx if pad_idx >= 0 else item_num - self.maxlen = maxlen - self.dev = device - self.init_std = init_std - - config = BertConfig( - vocab_size=item_num + 1, - hidden_size=embedding_dim, - num_hidden_layers=n_layers, - num_attention_heads=n_heads, - intermediate_size=embedding_dim * 4, - hidden_act="gelu", - hidden_dropout_prob=dropout, - attention_probs_dropout_prob=dropout, - max_position_embeddings=maxlen + 1, - initializer_range=init_std, - pad_token_id=self.pad_idx, - layer_norm_eps=1e-12, - use_cache=False, - ) - - self.item_emb = nn.Embedding( - num_embeddings=item_num + 1, - embedding_dim=embedding_dim, - padding_idx=self.pad_idx, - ) - self.transformer_model = BertModel(config) - self.item_biases = nn.Embedding(item_num + 1, 1, padding_idx=self.pad_idx) - self._init_weights() - self.to(device) - - def _init_weights(self): - self.item_emb.weight.data.normal_(mean=0.0, std=self.init_std) - self.item_emb.weight.data[self.pad_idx].zero_() - self.item_biases.weight.data.zero_() - - def _encode(self, hist_iids): - attention_mask = (hist_iids != self.pad_idx).long() - embeds = self.item_emb(hist_iids) - out = self.transformer_model( - inputs_embeds=embeds, attention_mask=attention_mask - ) - return out.last_hidden_state[:, -1, :] - - def forward(self, user_ids, hist_iids, out_iids, return_hidden=False): - hidden = self._encode(hist_iids) - item_e = self.item_emb(out_iids) - bias = self.item_biases(out_iids) - if return_hidden: - return hidden, item_e, bias - scores = torch.mm(hidden, item_e.T) + bias.T - return scores - - @torch.no_grad() - def predict(self, user_ids, log_seqs, item_indices=None): - if item_indices is None: - item_indices = torch.arange(self.item_num, device=self.dev) - else: - item_indices = torch.as_tensor( - item_indices, dtype=torch.long, device=self.dev - ) - if not isinstance(log_seqs, torch.Tensor): - log_seqs = torch.as_tensor(log_seqs, dtype=torch.long, device=self.dev) - hidden = self._encode(log_seqs) - item_e = self.item_emb(item_indices) - bias = self.item_biases(item_indices) - scores = torch.mm(hidden, item_e.T) + bias.T - return scores.squeeze().detach().cpu().numpy() diff --git a/cornac/models/bert4rec/recom_bert4rec.py b/cornac/models/bert4rec/recom_bert4rec.py deleted file mode 100644 index ac716280..00000000 --- a/cornac/models/bert4rec/recom_bert4rec.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import numpy as np -from tqdm.auto import trange - -from cornac.models.recommender import NextItemRecommender - -from ...utils import get_rng -from ..seq_utils import session_seq_iter, val_score - -SUPPORTED_LOSSES = ( - "bce", - "ce", - "bpr", - "bpr-max", - "softmax", - "cross-entropy", - "xe_softmax", - "top1", -) - - -class BERT4Rec(NextItemRecommender): - """BERT4Rec: a bidirectional transformer encoder for sequential rec. - - Wraps HuggingFace's :class:`~transformers.BertModel` as the sequence - encoder; the last-position hidden state scores candidate items by dot - product, sharing the ``(B, B+N)`` loss contract of - :mod:`cornac.models.seq_utils`. Parameters mirror - :class:`cornac.models.SASRec` (minus ``use_pos_emb`` — the backbone - provides its own positional embeddings); see the SASRec docstring for - details about ``loss``, ``model_selection``, and the rest. - - Note - ---- - This uses the next-item-at-last-position objective shared by the - transformer family in Cornac, *not* the canonical masked-language-model - (MLM) objective of the original paper. - - References - ---------- - Sun, F., Liu, J., Wu, J., Pei, C., Lin, X., Ou, W., & Jiang, P. (2019). - BERT4Rec: Sequential recommendation with bidirectional encoder - representations from transformer. CIKM. - """ - - def __init__( - self, - name="BERT4Rec", - embedding_dim=100, - loss="ce", - batch_size=512, - learning_rate=0.001, - n_sample=2048, - sample_alpha=0.5, - n_epochs=10, - max_len=50, - num_blocks=2, - num_heads=1, - dropout=0.2, - l2_reg=0.0, - bpreg=1.0, - elu_param=0.5, - device="cpu", - trainable=True, - verbose=False, - seed=None, - model_selection="last", - val_eval_every=5, - val_k=20, - val_metric="recall", - ): - super().__init__(name, trainable=trainable, verbose=verbose) - if loss not in SUPPORTED_LOSSES: - raise ValueError( - f"loss='{loss}' not supported; choose from {SUPPORTED_LOSSES}" - ) - if model_selection not in ("last", "best"): - raise ValueError( - f"model_selection='{model_selection}' not supported; choose 'last' or 'best'" - ) - self.embedding_dim = embedding_dim - self.loss = loss - self.batch_size = batch_size - self.learning_rate = learning_rate - self.n_sample = n_sample - self.sample_alpha = sample_alpha - self.n_epochs = n_epochs - self.max_len = max_len - self.num_blocks = num_blocks - self.num_heads = num_heads - self.dropout = dropout - self.l2_reg = l2_reg - self.bpreg = bpreg - self.elu_param = elu_param - self.device = device - self.seed = seed - self.rng = get_rng(seed) - self.model_selection = model_selection - self.val_eval_every = val_eval_every - self.val_k = val_k - self.val_metric = val_metric - - def fit(self, train_set, val_set=None): - super().fit(train_set, val_set) - if not self.trainable: - return self - - import torch - - from .bert4rec import BERT4RecModel - from ..seq_utils.losses import get_loss_function - - torch.manual_seed(self.seed if self.seed is not None else 0) - - self.pad_idx = self.total_items - self.model = BERT4RecModel( - item_num=self.total_items, - embedding_dim=self.embedding_dim, - maxlen=self.max_len, - n_layers=self.num_blocks, - n_heads=self.num_heads, - dropout=self.dropout, - pad_idx=self.pad_idx, - device=self.device, - ) - - loss_fn = get_loss_function(self.loss) - loss_kwargs = dict( - bpreg=self.bpreg, elu_param=self.elu_param, n_sample=self.n_sample - ) - opt = torch.optim.Adam( - self.model.parameters(), lr=self.learning_rate, betas=(0.9, 0.98) - ) - - best_val = -float("inf") - best_state = None - progress_bar = trange(1, self.n_epochs + 1, disable=not self.verbose) - for epoch_id in progress_bar: - self.model.train() - total_loss = 0.0 - cnt = 0 - for inc, (in_uids, hist_iids, out_iids) in enumerate( - session_seq_iter( - self.train_set, - pad_index=self.pad_idx, - batch_size=self.batch_size, - max_len=self.max_len, - n_sample=self.n_sample, - sample_alpha=self.sample_alpha, - rng=self.rng, - shuffle=True, - ) - ): - if len(hist_iids) < 2: - continue - hist_iids_t = torch.tensor( - hist_iids, dtype=torch.long, device=self.device, requires_grad=False - ) - out_iids_t = torch.tensor( - out_iids, dtype=torch.long, device=self.device, requires_grad=False - ) - - self.model.zero_grad() - item_scores = self.model(None, hist_iids_t, out_iids_t) - L = loss_fn( - item_scores, - out_iids=out_iids_t, - batch_size=len(hist_iids), - **loss_kwargs, - ) - if self.l2_reg > 0: - for p in self.model.parameters(): - L = L + self.l2_reg * torch.norm(p) - - L.backward() - opt.step() - - total_loss += L.cpu().detach().numpy() * len(hist_iids) - cnt += len(hist_iids) - if inc % 10 == 0 and cnt > 0: - progress_bar.set_postfix(loss=(total_loss / cnt)) - - if ( - self.model_selection == "best" - and val_set is not None - and epoch_id % self.val_eval_every == 0 - ): - score = val_score( - self, self.train_set, val_set, metric=self.val_metric, k=self.val_k - ) - if score is not None and score > best_val: - best_val = score - best_state = { - n: p.detach().clone() - for n, p in self.model.state_dict().items() - } - - if self.model_selection == "best" and best_state is not None: - self.model.load_state_dict(best_state) - return self - - def score(self, user_idx, history_items, **kwargs): - import torch - - if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") - log_seq = [self.pad_idx] * (self.max_len - len(history_items)) + list( - history_items - ) - log_seq = log_seq[-self.max_len :] - log_seq_t = torch.tensor([log_seq], dtype=torch.long, device=self.device) - self.model.eval() - return self.model.predict(user_idx, log_seq_t) diff --git a/cornac/models/bert4rec/requirements.txt b/cornac/models/bert4rec/requirements.txt deleted file mode 100644 index a808c3f6..00000000 --- a/cornac/models/bert4rec/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch>=1.12.0 -transformers>=4.30.0 diff --git a/cornac/models/gpt2rec/__init__.py b/cornac/models/gpt2rec/__init__.py deleted file mode 100644 index ed6d4676..00000000 --- a/cornac/models/gpt2rec/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -from .recom_gpt2rec import GPT2Rec diff --git a/cornac/models/gpt2rec/gpt2rec.py b/cornac/models/gpt2rec/gpt2rec.py deleted file mode 100644 index 5cde864c..00000000 --- a/cornac/models/gpt2rec/gpt2rec.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import torch -import torch.nn as nn - - -class GPT2RecModel(nn.Module): - """GPT2-based causal transformer for next-item recommendation. - - Same input/output contract as :class:`SASRecModel` and - :class:`BERT4RecModel`. Returns a ``(B, B+N)`` score matrix. - """ - - def __init__( - self, - item_num, - embedding_dim=100, - maxlen=20, - n_layers=2, - n_heads=1, - dropout=0.1, - pad_idx=-1, - init_std=0.02, - device="cpu", - ): - super().__init__() - from transformers.models.gpt2 import GPT2Config, GPT2Model - - self.item_num = item_num - self.pad_idx = pad_idx if pad_idx >= 0 else item_num - self.maxlen = maxlen - self.dev = device - self.init_std = init_std - - config = GPT2Config( - vocab_size=item_num + 1, - n_positions=maxlen + 1, - n_embd=embedding_dim, - n_layer=n_layers, - n_head=n_heads, - n_inner=embedding_dim * 4, - activation_function="gelu_new", - resid_pdrop=dropout, - embd_pdrop=dropout, - attn_pdrop=dropout, - initializer_range=init_std, - pad_token_id=self.pad_idx, - layer_norm_epsilon=1e-12, - use_cache=False, - ) - - self.item_emb = nn.Embedding( - item_num + 1, embedding_dim, padding_idx=self.pad_idx - ) - self.transformer_model = GPT2Model(config) - self.item_biases = nn.Embedding(item_num + 1, 1, padding_idx=self.pad_idx) - - self._init_weights() - self.to(device) - - def _init_weights(self): - self.item_emb.weight.data.normal_(mean=0.0, std=self.init_std) - self.item_emb.weight.data[self.pad_idx].zero_() - self.item_biases.weight.data.zero_() - - def _encode(self, hist_iids): - attention_mask = (hist_iids != self.pad_idx).long() - embeds = self.item_emb(hist_iids) - out = self.transformer_model( - inputs_embeds=embeds, attention_mask=attention_mask - ) - return out.last_hidden_state[:, -1, :] - - def forward(self, user_ids, hist_iids, out_iids, return_hidden=False): - hidden = self._encode(hist_iids) - item_e = self.item_emb(out_iids) - bias = self.item_biases(out_iids) - if return_hidden: - return hidden, item_e, bias - scores = torch.mm(hidden, item_e.T) + bias.T - return scores - - @torch.no_grad() - def predict(self, user_ids, log_seqs, item_indices=None): - if item_indices is None: - item_indices = torch.arange(self.item_num, device=self.dev) - else: - item_indices = torch.as_tensor( - item_indices, dtype=torch.long, device=self.dev - ) - if not isinstance(log_seqs, torch.Tensor): - log_seqs = torch.as_tensor(log_seqs, dtype=torch.long, device=self.dev) - hidden = self._encode(log_seqs) - item_e = self.item_emb(item_indices) - bias = self.item_biases(item_indices) - scores = torch.mm(hidden, item_e.T) + bias.T - return scores.squeeze().detach().cpu().numpy() diff --git a/cornac/models/gpt2rec/recom_gpt2rec.py b/cornac/models/gpt2rec/recom_gpt2rec.py deleted file mode 100644 index df8412c3..00000000 --- a/cornac/models/gpt2rec/recom_gpt2rec.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2026 The Cornac Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import numpy as np -from tqdm.auto import trange - -from cornac.models.recommender import NextItemRecommender - -from ...utils import get_rng -from ..seq_utils import session_seq_iter, val_score - -SUPPORTED_LOSSES = ( - "bce", - "ce", - "bpr", - "bpr-max", - "softmax", - "cross-entropy", - "xe_softmax", - "top1", -) - - -class GPT2Rec(NextItemRecommender): - """GPT2Rec: a causal (GPT-2) transformer for sequential recommendation. - - Wraps HuggingFace's :class:`~transformers.GPT2Model` as the sequence - encoder; the last-position hidden state scores candidate items by dot - product, sharing the ``(B, B+N)`` loss contract of - :mod:`cornac.models.seq_utils`. Parameters mirror - :class:`cornac.models.SASRec` (minus ``use_pos_emb`` — the backbone - provides its own positional embeddings); see the SASRec docstring for - details about ``loss``, ``model_selection``, and the rest. - - Note - ---- - This uses the next-item-at-last-position objective shared by the - transformer family in Cornac, *not* a canonical causal-language-model - (CLM) loss at every position. - - References - ---------- - de Souza Pereira Moreira, G., Rabhi, S., Lee, J. M., Ak, R., & Oldridge, E. - (2021). Transformers4Rec: Bridging the gap between NLP and sequential / - session-based recommendation. RecSys. - """ - - def __init__( - self, - name="GPT2Rec", - embedding_dim=100, - loss="ce", - batch_size=512, - learning_rate=0.001, - n_sample=2048, - sample_alpha=0.5, - n_epochs=10, - max_len=50, - num_blocks=2, - num_heads=1, - dropout=0.2, - l2_reg=0.0, - bpreg=1.0, - elu_param=0.5, - device="cpu", - trainable=True, - verbose=False, - seed=None, - model_selection="last", - val_eval_every=5, - val_k=20, - val_metric="recall", - ): - super().__init__(name, trainable=trainable, verbose=verbose) - if loss not in SUPPORTED_LOSSES: - raise ValueError( - f"loss='{loss}' not supported; choose from {SUPPORTED_LOSSES}" - ) - if model_selection not in ("last", "best"): - raise ValueError( - f"model_selection='{model_selection}' not supported; choose 'last' or 'best'" - ) - self.embedding_dim = embedding_dim - self.loss = loss - self.batch_size = batch_size - self.learning_rate = learning_rate - self.n_sample = n_sample - self.sample_alpha = sample_alpha - self.n_epochs = n_epochs - self.max_len = max_len - self.num_blocks = num_blocks - self.num_heads = num_heads - self.dropout = dropout - self.l2_reg = l2_reg - self.bpreg = bpreg - self.elu_param = elu_param - self.device = device - self.seed = seed - self.rng = get_rng(seed) - self.model_selection = model_selection - self.val_eval_every = val_eval_every - self.val_k = val_k - self.val_metric = val_metric - - def fit(self, train_set, val_set=None): - super().fit(train_set, val_set) - if not self.trainable: - return self - - import torch - - from .gpt2rec import GPT2RecModel - from ..seq_utils.losses import get_loss_function - - torch.manual_seed(self.seed if self.seed is not None else 0) - - self.pad_idx = self.total_items - self.model = GPT2RecModel( - item_num=self.total_items, - embedding_dim=self.embedding_dim, - maxlen=self.max_len, - n_layers=self.num_blocks, - n_heads=self.num_heads, - dropout=self.dropout, - pad_idx=self.pad_idx, - device=self.device, - ) - - loss_fn = get_loss_function(self.loss) - loss_kwargs = dict( - bpreg=self.bpreg, elu_param=self.elu_param, n_sample=self.n_sample - ) - opt = torch.optim.Adam( - self.model.parameters(), lr=self.learning_rate, betas=(0.9, 0.98) - ) - - best_val = -float("inf") - best_state = None - progress_bar = trange(1, self.n_epochs + 1, disable=not self.verbose) - for epoch_id in progress_bar: - self.model.train() - total_loss = 0.0 - cnt = 0 - for inc, (in_uids, hist_iids, out_iids) in enumerate( - session_seq_iter( - self.train_set, - pad_index=self.pad_idx, - batch_size=self.batch_size, - max_len=self.max_len, - n_sample=self.n_sample, - sample_alpha=self.sample_alpha, - rng=self.rng, - shuffle=True, - ) - ): - if len(hist_iids) < 2: - continue - hist_iids_t = torch.tensor( - hist_iids, dtype=torch.long, device=self.device, requires_grad=False - ) - out_iids_t = torch.tensor( - out_iids, dtype=torch.long, device=self.device, requires_grad=False - ) - - self.model.zero_grad() - item_scores = self.model(None, hist_iids_t, out_iids_t) - L = loss_fn( - item_scores, - out_iids=out_iids_t, - batch_size=len(hist_iids), - **loss_kwargs, - ) - if self.l2_reg > 0: - for p in self.model.parameters(): - L = L + self.l2_reg * torch.norm(p) - - L.backward() - opt.step() - - total_loss += L.cpu().detach().numpy() * len(hist_iids) - cnt += len(hist_iids) - if inc % 10 == 0 and cnt > 0: - progress_bar.set_postfix(loss=(total_loss / cnt)) - - if ( - self.model_selection == "best" - and val_set is not None - and epoch_id % self.val_eval_every == 0 - ): - score = val_score( - self, self.train_set, val_set, metric=self.val_metric, k=self.val_k - ) - if score is not None and score > best_val: - best_val = score - best_state = { - n: p.detach().clone() - for n, p in self.model.state_dict().items() - } - - if self.model_selection == "best" and best_state is not None: - self.model.load_state_dict(best_state) - return self - - def score(self, user_idx, history_items, **kwargs): - import torch - - if len(history_items) == 0: - return np.ones(self.total_items, dtype="float") - log_seq = [self.pad_idx] * (self.max_len - len(history_items)) + list( - history_items - ) - log_seq = log_seq[-self.max_len :] - log_seq_t = torch.tensor([log_seq], dtype=torch.long, device=self.device) - self.model.eval() - return self.model.predict(user_idx, log_seq_t) diff --git a/cornac/models/gpt2rec/requirements.txt b/cornac/models/gpt2rec/requirements.txt deleted file mode 100644 index a808c3f6..00000000 --- a/cornac/models/gpt2rec/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -torch>=1.12.0 -transformers>=4.30.0 From 75db24c8100682fb55a7b72f92df58929d3e1ff9 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Sun, 5 Jul 2026 20:12:31 +0800 Subject: [PATCH 3/5] update example --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 0228cdb6..7fc06d79 100644 --- a/examples/README.md +++ b/examples/README.md @@ -128,7 +128,7 @@ [fpmc_diginetica.py](fpmc_diginetica.py) - Example of Factorizing Personalized Markov Chains (FPMC) with Diginetica dataset. -[transformer_rec_diginetica.py](transformer_rec_diginetica.py) - Example of Transformer-based Recommendation models (SASRec, BERT4Rec, GPT2Rec) with Diginetica dataset. +[transformer_rec_diginetica.py](transformer_rec_diginetica.py) - Example of Transformer-based Recommendation models (SASRec, TransformerRec with CLM/MLM objectives) with Diginetica dataset. ---- From 41977eefb7c90b62506f608605d58de180bd7611 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Mon, 6 Jul 2026 16:41:22 +0800 Subject: [PATCH 4/5] add BERT4Rec class interface --- README.md | 1 + cornac/models/__init__.py | 1 + cornac/models/bert4rec/__init__.py | 16 ++++ cornac/models/bert4rec/recom_bert4rec.py | 96 ++++++++++++++++++++++++ cornac/models/bert4rec/requirements.txt | 2 + 5 files changed, 116 insertions(+) create mode 100644 cornac/models/bert4rec/__init__.py create mode 100644 cornac/models/bert4rec/recom_bert4rec.py create mode 100644 cornac/models/bert4rec/requirements.txt diff --git a/README.md b/README.md index b24439d5..18cb586f 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,7 @@ The table below lists the recommendation models/algorithms featured in Cornac. E | | [Temporal-Item-Frequency-based User-KNN (TIFUKNN)](cornac/models/tifuknn), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.tifuknn.recom_tifuknn), [paper](https://arxiv.org/pdf/2006.00556.pdf) | Next-Basket | CPU | [quick-start](examples/tifuknn_tafeng.py) | | [Variational Autoencoder for Top-N Recommendations (RecVAE)](cornac/models/recvae), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.recvae.recom_recvae), [paper](https://doi.org/10.1145/3336191.3371831) | Collaborative Filtering | [requirements](cornac/models/recvae/requirements.txt), CPU / GPU | [quick-start](examples/recvae_example.py) | 2019 | [Correlation-Sensitive Next-Basket Recommendation (Beacon)](cornac/models/beacon), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#correlation-sensitive-next-basket-recommendation-beacon), [paper](https://www.ijcai.org/proceedings/2019/0389.pdf) | Next-Basket | [requirements](cornac/models/beacon/requirements.txt), CPU / GPU | [quick-start](examples/beacon_tafeng.py) +| | [BERT4Rec: Sequential Recommendation with Bidirectional Encoder Representations from Transformer (BERT4Rec)](cornac/models/bert4rec), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.bert4rec.recom_bert4rec), [paper](https://arxiv.org/pdf/1904.06690.pdf) | Next-Item | [requirements](cornac/models/bert4rec/requirements.txt), CPU / GPU | [quick-start](examples/transformer_rec_diginetica.py) | | [Embarrassingly Shallow Autoencoders for Sparse Data (EASEᴿ)](cornac/models/ease), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.ease.recom_ease), [paper](https://arxiv.org/pdf/1905.03375.pdf) | Collaborative Filtering | CPU | [quick-start](examples/ease_movielens.py) | | [Neural Graph Collaborative Filtering (NGCF)](cornac/models/ngcf), [docs](https://cornac.readthedocs.io/en/stable/api_ref/models.html#module-cornac.models.ngcf.recom_ngcf), [paper](https://arxiv.org/pdf/1905.08108.pdf) | Collaborative Filtering | [requirements](cornac/models/ngcf/requirements.txt), CPU / GPU | [quick-start](examples/ngcf_example.py) | | [Sampler Design for Bayesian Personalized Ranking by Leveraging View Data (VEBPR)](cornac/models/bpr), [paper](https://arxiv.org/pdf/1809.08162) | Collaborative Filtering | CPU | [quick-start](examples/vebpr_example.py) diff --git a/cornac/models/__init__.py b/cornac/models/__init__.py index 50f6c145..b20c7494 100644 --- a/cornac/models/__init__.py +++ b/cornac/models/__init__.py @@ -24,6 +24,7 @@ from .ann import ScaNNANN from .baseline_only import BaselineOnly from .beacon import Beacon +from .bert4rec import BERT4Rec from .bivaecf import BiVAECF from .bpr import BPR from .bpr import WBPR diff --git a/cornac/models/bert4rec/__init__.py b/cornac/models/bert4rec/__init__.py new file mode 100644 index 00000000..af46ce60 --- /dev/null +++ b/cornac/models/bert4rec/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from .recom_bert4rec import BERT4Rec diff --git a/cornac/models/bert4rec/recom_bert4rec.py b/cornac/models/bert4rec/recom_bert4rec.py new file mode 100644 index 00000000..c733171f --- /dev/null +++ b/cornac/models/bert4rec/recom_bert4rec.py @@ -0,0 +1,96 @@ +# Copyright 2026 The Cornac Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +from ..transformer_rec import TransformerRec + + +class BERT4Rec(TransformerRec): + """BERT4Rec: a bidirectional transformer encoder for sequential rec. + + A light interface over :class:`~cornac.models.TransformerRec` fixed to + ``backbone='bert', objective='clm', loss_at='last'``, i.e. the + next-item-at-last-position training shared by the transformer family in + Cornac. See the :class:`~cornac.models.TransformerRec` docstring for the + shared parameters (``loss``, ``model_selection``, negative sampling, + etc.). + + Note + ---- + The original paper trains with the masked-language-model (Cloze) objective; + that canonical setting is available as + ``TransformerRec(backbone='bert', objective='mlm')``. + In our experiments, the last-position objective gives better performance. + + References + ---------- + Sun, F., Liu, J., Wu, J., Pei, C., Lin, X., Ou, W., & Jiang, P. (2019). + BERT4Rec: Sequential recommendation with bidirectional encoder + representations from transformer. CIKM. + """ + + def __init__( + self, + name="BERT4Rec", + embedding_dim=100, + loss="ce", + batch_size=512, + learning_rate=0.001, + n_sample=2048, + sample_alpha=0.5, + n_epochs=10, + max_len=50, + num_blocks=2, + num_heads=1, + dropout=0.2, + l2_reg=0.0, + bpreg=1.0, + elu_param=0.5, + device="cpu", + trainable=True, + verbose=False, + seed=None, + model_selection="last", + val_eval_every=5, + val_k=20, + val_metric="recall", + ): + super().__init__( + name=name, + backbone="bert", + objective="clm", + loss_at="last", + embedding_dim=embedding_dim, + loss=loss, + batch_size=batch_size, + learning_rate=learning_rate, + n_sample=n_sample, + sample_alpha=sample_alpha, + n_epochs=n_epochs, + max_len=max_len, + num_blocks=num_blocks, + num_heads=num_heads, + dropout=dropout, + l2_reg=l2_reg, + bpreg=bpreg, + elu_param=elu_param, + device=device, + trainable=trainable, + verbose=verbose, + seed=seed, + model_selection=model_selection, + val_eval_every=val_eval_every, + val_k=val_k, + val_metric=val_metric, + ) diff --git a/cornac/models/bert4rec/requirements.txt b/cornac/models/bert4rec/requirements.txt new file mode 100644 index 00000000..a808c3f6 --- /dev/null +++ b/cornac/models/bert4rec/requirements.txt @@ -0,0 +1,2 @@ +torch>=1.12.0 +transformers>=4.30.0 From 8c6b5fe0dda091cffe94647681efcb80995f4671 Mon Sep 17 00:00:00 2001 From: hieuddo Date: Mon, 6 Jul 2026 16:43:02 +0800 Subject: [PATCH 5/5] move reusable code to base --- .../models/transformer_rec/objectives/base.py | 58 +++++++++++++++++++ .../models/transformer_rec/objectives/clm.py | 20 +------ .../models/transformer_rec/objectives/mlm.py | 23 +++----- .../models/transformer_rec/objectives/plm.py | 33 +---------- .../models/transformer_rec/objectives/rtd.py | 23 +------- 5 files changed, 74 insertions(+), 83 deletions(-) diff --git a/cornac/models/transformer_rec/objectives/base.py b/cornac/models/transformer_rec/objectives/base.py index 1135974d..31058a93 100644 --- a/cornac/models/transformer_rec/objectives/base.py +++ b/cornac/models/transformer_rec/objectives/base.py @@ -18,8 +18,66 @@ positions, and targets; runs the model; and returns a scalar loss computed by one of the shared :mod:`cornac.models.seq_utils` loss functions against a ``(M, M + N)`` score matrix. + +Shared helpers used by several objectives live here too: +:func:`bernoulli_mask` (MLM / PLM / RTD position selection) and +:func:`build_out_iids` (positives + sampled negatives concatenation). """ +import numpy as np +import torch + + +def bernoulli_mask(non_pad, mask_prob, rng): + """Bernoulli position selection with the T4R MLM safeguards. + + Draws i.i.d. Bernoulli(``mask_prob``) over the non-pad positions, then + guarantees per row at least one selected position and, when the row has + more than one non-pad item, at least one visible (unselected) item. + + Parameters + ---------- + non_pad : numpy.ndarray of bool, shape (B, T) + True where the position holds a real item. + mask_prob : float + Per-position selection probability. + rng : numpy.random.RandomState + + Returns + ------- + numpy.ndarray of bool, shape (B, T) + True at the selected positions. + """ + B, T = non_pad.shape + mask_labels = (rng.random((B, T)) < mask_prob) & non_pad + for b in range(B): + valid = np.where(non_pad[b])[0] + if len(valid) == 0: + continue + if not mask_labels[b].any(): + mask_labels[b, rng.choice(valid)] = True + # If every non-pad item is selected, unselect one to keep context. + if len(valid) > 1 and mask_labels[b].sum() == len(valid): + masked = np.where(mask_labels[b])[0] + mask_labels[b, rng.choice(masked)] = False + return mask_labels + + +def build_out_iids(target_flat, sample_negatives, loss_kwargs, device): + """Concatenate positives with sampled negatives. + + The negative count is read from ``loss_kwargs['n_sample']`` (mirroring + the family's ``loss_kwargs = dict(..., n_sample=...)`` convention); + absent or zero yields in-batch negatives only. + """ + n_sample = loss_kwargs.get("n_sample", 0) + if sample_negatives is None or not n_sample: + return target_flat + negatives = torch.as_tensor( + sample_negatives(n_sample), dtype=torch.long, device=device + ) + return torch.cat([target_flat, negatives]) + class Objective: """Base training objective (next-item / causal defaults).""" diff --git a/cornac/models/transformer_rec/objectives/clm.py b/cornac/models/transformer_rec/objectives/clm.py index 7ef06853..a3d8896b 100644 --- a/cornac/models/transformer_rec/objectives/clm.py +++ b/cornac/models/transformer_rec/objectives/clm.py @@ -15,7 +15,7 @@ import torch -from .base import Objective +from .base import Objective, build_out_iids class CLMObjective(Objective): @@ -47,7 +47,7 @@ def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): hidden_flat = hidden[valid] # (M, D) target_flat = targets[valid] # (M,) - out_iids = _build_out_iids( + out_iids = build_out_iids( target_flat, sample_negatives, loss_kwargs, model.dev ) scores = model.score_positions(hidden_flat, out_iids) @@ -77,19 +77,3 @@ def compute_loss_prefix(self, model, hist_iids, out_iids, loss_fn, loss_kwargs): batch_size=hist_iids.size(0), **loss_kwargs, ) - - -def _build_out_iids(target_flat, sample_negatives, loss_kwargs, device): - """Concatenate positives with sampled negatives. - - The negative count is read from ``loss_kwargs['n_sample']`` (mirroring - the family's ``loss_kwargs = dict(..., n_sample=...)`` convention); - absent or zero yields in-batch negatives only. - """ - n_sample = loss_kwargs.get("n_sample", 0) - if sample_negatives is None or not n_sample: - return target_flat - negatives = torch.as_tensor( - sample_negatives(n_sample), dtype=torch.long, device=device - ) - return torch.cat([target_flat, negatives]) diff --git a/cornac/models/transformer_rec/objectives/mlm.py b/cornac/models/transformer_rec/objectives/mlm.py index ffb6196f..4691cde0 100644 --- a/cornac/models/transformer_rec/objectives/mlm.py +++ b/cornac/models/transformer_rec/objectives/mlm.py @@ -16,8 +16,7 @@ import numpy as np import torch -from .base import Objective -from .clm import _build_out_iids +from .base import Objective, bernoulli_mask, build_out_iids class MLMObjective(Objective): @@ -25,7 +24,9 @@ class MLMObjective(Objective): Random non-pad positions are replaced by ``mask_idx`` and the model must recover the original items. Masking always replaces with the mask token - (no 80/10/10 split). Requires a bidirectional backbone. + (no 80/10/10 split) and follows the shared :func:`bernoulli_mask` + safeguards (at least one masked and, where possible, at least one + visible item per row). Requires a bidirectional backbone. """ VALID_ATTENTION = ("bidirectional",) @@ -38,18 +39,10 @@ def __init__(self, pad_idx, mask_idx, rng, mask_prob=0.2): def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): seqs_t = torch.as_tensor(seqs, dtype=torch.long, device=model.dev) seqs_np = np.asarray(seqs) - B, T = seqs_np.shape - # Sample mask positions among NON-pad positions; force >= 1 per row. - mask_matrix = np.zeros((B, T), dtype=bool) - for b in range(B): - positions = np.nonzero(seqs_np[b] != self.pad_idx)[0] - if len(positions) == 0: - continue - chosen = positions[self.rng.rand(len(positions)) < self.mask_prob] - if len(chosen) == 0: - chosen = self.rng.choice(positions, size=1) - mask_matrix[b, chosen] = True + mask_matrix = bernoulli_mask( + seqs_np != self.pad_idx, self.mask_prob, self.rng + ) mask_t = torch.as_tensor(mask_matrix, device=model.dev) inputs = seqs_t.clone() @@ -59,7 +52,7 @@ def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): hidden_flat = hidden[mask_t] # (M, D) target_flat = seqs_t[mask_t] # (M,) - out_iids = _build_out_iids( + out_iids = build_out_iids( target_flat, sample_negatives, loss_kwargs, model.dev ) scores = model.score_positions(hidden_flat, out_iids) diff --git a/cornac/models/transformer_rec/objectives/plm.py b/cornac/models/transformer_rec/objectives/plm.py index 98783e43..228b8908 100644 --- a/cornac/models/transformer_rec/objectives/plm.py +++ b/cornac/models/transformer_rec/objectives/plm.py @@ -16,7 +16,7 @@ import numpy as np import torch -from .base import Objective +from .base import Objective, bernoulli_mask, build_out_iids class PLMObjective(Objective): @@ -73,31 +73,11 @@ def __init__(self, pad_idx, mask_idx, rng, mask_prob=0.2): super().__init__(pad_idx, mask_idx, rng) self.mask_prob = mask_prob - def _select_targets(self, non_pad): - """Bernoulli target selection over non-pad positions. - - Guarantees at least one target and at least one visible (non-target) - item per row, following the T4R MLM safeguards. - """ - B, T = non_pad.shape - mask_labels = (self.rng.random((B, T)) < self.mask_prob) & non_pad - for b in range(B): - valid = np.where(non_pad[b])[0] - if len(valid) == 0: - continue - if not mask_labels[b].any(): - mask_labels[b, self.rng.choice(valid)] = True - # If every non-pad item is a target, unmask one to keep context. - if len(valid) > 1 and mask_labels[b].sum() == len(valid): - masked = np.where(mask_labels[b])[0] - mask_labels[b, self.rng.choice(masked)] = False - return mask_labels - def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): device = model.dev B, T = seqs.shape non_pad = seqs != self.pad_idx - mask_labels = self._select_targets(non_pad) + mask_labels = bernoulli_mask(non_pad, self.mask_prob, self.rng) # Random factorization-order attention mask (T4R recipe). perm_mask = np.zeros((B, T, T), dtype=np.float32) @@ -135,14 +115,7 @@ def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): ] # (M,) M = targets.shape[0] - out_iids = targets - n_neg = loss_kwargs.get("n_sample", 0) or 0 - if sample_negatives is not None and n_neg > 0: - negs = torch.as_tensor( - sample_negatives(n_neg), dtype=torch.long, device=device - ) - out_iids = torch.cat([targets, negs]) - + out_iids = build_out_iids(targets, sample_negatives, loss_kwargs, device) scores = model.score_positions(hidden, out_iids) # (M, M+N) return loss_fn(scores, out_iids=out_iids, batch_size=M, **loss_kwargs) diff --git a/cornac/models/transformer_rec/objectives/rtd.py b/cornac/models/transformer_rec/objectives/rtd.py index d3546c1e..269ee0be 100644 --- a/cornac/models/transformer_rec/objectives/rtd.py +++ b/cornac/models/transformer_rec/objectives/rtd.py @@ -13,13 +13,11 @@ # limitations under the License. # ============================================================================ -import numpy as np import torch import torch.nn as nn import torch.nn.functional as F -from .base import Objective -from .clm import _build_out_iids +from .base import Objective, bernoulli_mask, build_out_iids class RTDObjective(Objective): @@ -105,25 +103,10 @@ def build(self, model, device): def parameters(self): return list(self.disc_head.parameters()) - def _mlm_mask(self, non_pad): - """Bernoulli MLM masking with T4R's at-least-one / not-all safeguards.""" - B, T = non_pad.shape - mask_labels = (self.rng.random((B, T)) < self.mask_prob) & non_pad - for b in range(B): - valid = np.where(non_pad[b])[0] - if len(valid) == 0: - continue - if not mask_labels[b].any(): - mask_labels[b, self.rng.choice(valid)] = True - if len(valid) > 1 and mask_labels[b].sum() == len(valid): - masked = np.where(mask_labels[b])[0] - mask_labels[b, self.rng.choice(masked)] = False - return mask_labels - def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): device = model.dev non_pad = seqs != self.pad_idx - mask_labels = self._mlm_mask(non_pad) + mask_labels = bernoulli_mask(non_pad, self.mask_prob, self.rng) seqs_t = torch.as_tensor(seqs, dtype=torch.long, device=device) non_pad_t = torch.as_tensor(non_pad, device=device) @@ -136,7 +119,7 @@ def compute_loss(self, model, seqs, sample_negatives, loss_fn, loss_kwargs): mlm_hidden = hidden[mask_t] # (M, D) targets = seqs_t[mask_t] # (M,) - out_iids = _build_out_iids(targets, sample_negatives, loss_kwargs, device) + out_iids = build_out_iids(targets, sample_negatives, loss_kwargs, device) mlm_scores = model.score_positions(mlm_hidden, out_iids) # (M, M+N) mlm_loss = loss_fn( mlm_scores, out_iids=out_iids, batch_size=targets.shape[0], **loss_kwargs