Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,14 @@ namespace fk {
}

FK_HOST_DEVICE_FUSE uint num_elems_z(const Point thread, const OperationDataType& opData) {
return BATCH;
// Report the planes of the view we were actually given, not the compile-time BATCH.
// BATCH stays the modulus used by computeCircularThreadIdx for the circular wrap-around,
// but it is not necessarily the number of planes this read is responsible for:
// CircularTensor::update narrows the copy sequence to BATCH - 1 planes so that the plane
// space of the DivergentBatchTransformDPP (the sum of the planes declared by each
// sequence) adds up to BATCH. Returning BATCH here made that total BATCH + 1 and let a
// thread address a plane past the end of the Tensor.
return Operation::num_elems_z(thread, opData.params.opData);
}

FK_HOST_DEVICE_FUSE uint pitch(const Point thread, const OperationDataType& opData) {
Expand Down
19 changes: 18 additions & 1 deletion include/fused_kernel/core/data/circular_tensor.h
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,15 @@ namespace fk {
}
using equivalentReadDFType = EquivalentType_t<writeDFType, WriteInstantiableOperations, ReadInstantiableOperations>;

// The DivergentBatchTransformDPP defines a global plane space whose size is the sum of the
// planes declared by each sequence. Both sequences write into the same output Tensor, but
// each one owns a different subset of its planes: the update sequence produces a single
// plane (the newly inserted image) and the copy sequence produces the remaining BATCH - 1.
// Therefore each sequence must be given a view of the data that reports its own number of
// planes, so that the total adds up to BATCH and no thread addresses a plane past the end.
// The update sequence already declares a single plane through its 2D read.
constexpr uint COPY_PLANES = static_cast<uint>(BATCH) - 1u;

MidWrite<CircularTensorWrite<CircularDirection::Ascendent, writeOpType, BATCH>> updateWriteToTemp;
updateWriteToTemp.params.first = m_nextUpdateIdx;
updateWriteToTemp.params.opData.params = m_tempTensor.ptr();
Expand All @@ -130,8 +139,16 @@ namespace fk {
equivalentReadDFType nonUpdateRead;
nonUpdateRead.params.first = m_nextUpdateIdx;
nonUpdateRead.params.opData.params = m_tempTensor.ptr();
nonUpdateRead.params.opData.params.dims.planes = COPY_PLANES;

// The copy sequence writes into the same output Tensor as the update sequence, using the
// global plane index, so it needs the full view: narrowing dims.planes here would be
// misleading. Write operations take no part in the thread space (the DPP derives it from
// the first IOp of the sequence, i.e. the read) and TensorSplit/TensorWrite address the
// data through the pitch/plane_pitch strides, never through dims.planes.
const auto nonUpdateWrite = writeInstantiableOperation;

const auto copyOps = buildOperationSequence(nonUpdateRead, writeInstantiableOperation);
const auto copyOps = buildOperationSequence(nonUpdateRead, nonUpdateWrite);

if (PA == ParArch::GPU_NVIDIA && !(this->type == MemType::Device || this->type == MemType::DeviceAndPinned)) {
throw std::runtime_error("CircularTensor operations on Device memory only supported \
Expand Down
86 changes: 66 additions & 20 deletions include/fused_kernel/core/execution_model/data_parallel_patterns.h
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,18 @@ namespace fk { // namespace FusedKernel
return Parent::getActiveThreads(details, iOp);
}

// Executes the work of a single thread. The caller is responsible for providing
// the thread coordinates, which allows other DPPs (like DivergentBatchTransformDPP)
// to reuse this implementation while owning the index generation.
template <typename... IOps>
FK_DEVICE_FUSE void exec_thread(const Point& thread, const Details& details, const IOps&... iOps) {
const ActiveThreads activeThreads = getActiveThreads(details, get_arg<0>(iOps...));

if (thread.x < activeThreads.x && thread.y < activeThreads.y) {
Parent::execute_thread(thread, activeThreads, iOps...);
}
}

template <typename... IOps>
FK_DEVICE_FUSE void exec(const Details& details, const IOps&... iOps) {
const cg::thread_block g = cg::this_thread_block();
Expand All @@ -226,11 +238,7 @@ namespace fk { // namespace FusedKernel
const int z = g.group_index().z; // So far we only consider the option of using the z dimension to specify n (x*y) thread planes
const Point thread{ x, y, z };

const ActiveThreads activeThreads = getActiveThreads(details, get_arg<0>(iOps...));

if (x < activeThreads.x && y < activeThreads.y) {
Parent::execute_thread(thread, activeThreads, iOps...);
}
exec_thread(thread, details, iOps...);
}
};
#endif // defined(__NVCC__)
Expand All @@ -248,16 +256,27 @@ namespace fk { // namespace FusedKernel
return Parent::getActiveThreads(details, iOp);
}

// Executes the work of a single thread. The caller is responsible for providing
// the thread coordinates, which allows other DPPs (like DivergentBatchTransformDPP)
// to reuse this implementation while owning the index generation.
template <typename... IOps>
FK_HOST_FUSE void exec_thread(const Point& thread, const Details& details, const IOps&... iOps) {
const ActiveThreads activeThreads = getActiveThreads(details, get_arg<0>(iOps...));

if (thread.x < activeThreads.x && thread.y < activeThreads.y) {
Parent::execute_thread(thread, activeThreads, iOps...);
}
}

template <typename... IOps>
FK_HOST_FUSE void exec(const Details& details, const IOps&... iOps) {
using TFI = typename Details::TFI;
const ActiveThreads activeThreads = getActiveThreads(details, get_arg<0>(iOps...));

for (int z = 0; z < activeThreads.z; ++z) {
for (int y = 0; y < activeThreads.y; ++y) {
Comment thread
morousg marked this conversation as resolved.
for (int x = 0; x < activeThreads.x; ++x) {
const Point thread{ x, y, z };
Parent::execute_thread(thread, activeThreads, iOps...);
exec_thread(thread, details, iOps...);
}
}
}
Expand All @@ -267,25 +286,47 @@ namespace fk { // namespace FusedKernel
template <enum ParArch PA, typename SequenceSelector>
struct DivergentBatchTransformDPP;

template <typename SequenceSelector>
template <enum ParArch PA, typename SequenceSelector>
struct DivergentBatchTransformDPPBase {
friend struct DivergentBatchTransformDPP<ParArch::GPU_NVIDIA, SequenceSelector>; // Allow DivergentBatchTransformDPP to access private members
friend struct DivergentBatchTransformDPP<ParArch::CPU, SequenceSelector>; // Allow DivergentBatchTransformDPPBase to access private members
private:
template <typename... IOps>
FK_HOST_DEVICE_FUSE void launchTransformDPP(const IOps&... iOps) {
FK_HOST_DEVICE_FUSE void launchTransformDPP(const Point& thread, const IOps&... iOps) {
using Details = TransformDPPDetails<false, IOps...>;
TransformDPP<ParArch::GPU_NVIDIA, TF::DISABLED, Details, true>::exec(Details{}, iOps...);
using TDPP = TransformDPP<PA, TF::DISABLED, Details, true>;
if constexpr (PA == ParArch::CPU) {
// On CPU there is no thread grid: the x and y indices are generated here,
// from the geometry of this sequence, while the plane index comes from the caller.
const ActiveThreads activeThreads = TDPP::getActiveThreads(Details{}, get_arg<0>(iOps...));
for (int y = 0; y < static_cast<int>(activeThreads.y); ++y) {
for (int x = 0; x < static_cast<int>(activeThreads.x); ++x) {
TDPP::exec_thread(Point{ x, y, thread.z }, Details{}, iOps...);
}
}
} else {
TDPP::exec_thread(thread, Details{}, iOps...);
}
}

// Functor used to expand the IOp tuple of an operation sequence while carrying
// the thread coordinates, which are not part of the tuple.
template <typename... IOps>
struct LaunchTransformDPPForThread {
Point thread;
FK_HOST_DEVICE_CNST void operator()(const IOps&... iOps) const {
launchTransformDPP(thread, iOps...);
}
};

template <int OpSequenceNumber, typename... IOps, typename... IOpSequenceTypes>
FK_HOST_DEVICE_FUSE void divergent_operate(const uint z,
FK_HOST_DEVICE_FUSE void divergent_operate(const Point& thread,
const InstantiableOperationSequence<IOps...>& iOpSequence,
const IOpSequenceTypes&... iOpSequences) {
if (OpSequenceNumber == SequenceSelector::at(z)) {
apply_d(launchTransformDPP<IOps...>, iOpSequence.iOps);
if (OpSequenceNumber == SequenceSelector::at(thread.z)) {
apply_d(LaunchTransformDPPForThread<IOps...>{ thread }, iOpSequence.iOps);
} else if constexpr (sizeof...(iOpSequences) > 0) {
divergent_operate<OpSequenceNumber + 1>(z, iOpSequences...);
divergent_operate<OpSequenceNumber + 1>(thread, iOpSequences...);
}
}
};
Expand All @@ -305,31 +346,36 @@ namespace fk { // namespace FusedKernel
template <typename SequenceSelector>
struct DivergentBatchTransformDPP<ParArch::GPU_NVIDIA, SequenceSelector> {
private:
using Parent = DivergentBatchTransformDPPBase<SequenceSelector>;
using Parent = DivergentBatchTransformDPPBase<ParArch::GPU_NVIDIA, SequenceSelector>;
public:
using DPPDetails = DivergentBatchTransformDPPDetails<ParArch::GPU_NVIDIA>;
static constexpr ParArch PAR_ARCH = ParArch::GPU_NVIDIA;
template <typename... IOpSequenceTypes>
FK_DEVICE_FUSE void exec(const DPPDetails&, const IOpSequenceTypes&... iOpSequences) {

const cg::thread_block g = cg::this_thread_block();
const uint z = g.group_index().z;

Parent::template divergent_operate<0>(z, iOpSequences...);
const int x = (g.dim_threads().x * g.group_index().x) + g.thread_index().x;
const int y = (g.dim_threads().y * g.group_index().y) + g.thread_index().y;
const int z = g.group_index().z;
const Point thread{ x, y, z };

Parent::template divergent_operate<0>(thread, iOpSequences...);
}
};
#endif // defined(__NVCC__)
template <typename SequenceSelector>
struct DivergentBatchTransformDPP<ParArch::CPU, SequenceSelector> {
private:
using Parent = DivergentBatchTransformDPPBase<SequenceSelector>;
using Parent = DivergentBatchTransformDPPBase<ParArch::CPU, SequenceSelector>;
public:
using DPPDetails = DivergentBatchTransformDPPDetails<ParArch::CPU>;
static constexpr ParArch PAR_ARCH = ParArch::CPU;
template <typename... IOpSequenceTypes>
FK_DEVICE_FUSE void exec(const DPPDetails& details, const IOpSequenceTypes&... iOpSequences) {
Comment on lines 374 to 375
for (uint z = 0; z < details.numPlanes; ++z) {
Parent::template divergent_operate<0>(z, iOpSequences...);
for (int z = 0; z < static_cast<int>(details.numPlanes); ++z) {
const Point thread{ 0, 0, z };
Parent::template divergent_operate<0>(thread, iOpSequences...);
}
}
};
Expand Down
50 changes: 50 additions & 0 deletions include/fused_kernel/core/execution_model/executors.h
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,56 @@ FK_HOST_FUSE void executeOperations(const std::array<Ptr2D<I>, Batch>& input, co
DECLARE_EXECUTOR_PARENT_IMPL
};

template <typename SequenceSelector>
struct Executor<DivergentBatchTransformDPP<ParArch::CPU, SequenceSelector>> {
private:
using DPPType = DivergentBatchTransformDPP<ParArch::CPU, SequenceSelector>;
using DPPDetails = typename DPPType::DPPDetails;
using SelfType = Executor<DPPType>;

template <typename... IOpSequenceTypes>
FK_HOST_FUSE ActiveThreads getActiveThreads(const IOpSequenceTypes &...iOpSequences) {
const uint x = cxp::max::f(get<0>(iOpSequences.iOps).getActiveThreads().x...);
const uint y = cxp::max::f(get<0>(iOpSequences.iOps).getActiveThreads().y...);
const uint z = cxp::sum::f(get<0>(iOpSequences.iOps).getActiveThreads().z...);
return ActiveThreads{x, y, z};
}

template <typename... IOps>
FK_HOST_FUSE auto fuseBackSequence(const IOpSequence<IOps...> &iOpSeq) {
return buildOperationSequence_tup(apply(
[](auto &&...args) {
// Now fuse_back deduces the types naturally and preserves value categories via perfect forwarding
return BackFuser::fuse_back(std::forward<decltype(args)>(args)...);
},
iOpSeq.iOps));
}

template <typename... IOpSequenceTypes>
FK_HOST_FUSE void executeOperationsFused(Stream_<ParArch::CPU> &stream,
const IOpSequenceTypes &...iOpSequences) {
const ActiveThreads activeThreads = getActiveThreads(iOpSequences...);
const DPPDetails details{ .numPlanes = activeThreads.z };

DivergentBatchTransformDPP<ParArch::CPU, SequenceSelector>::exec(details, iOpSequences...);
}

template <typename... IOpSequenceTypes>
FK_HOST_FUSE void executeOperations_helper(Stream_<ParArch::CPU> &stream,
const IOpSequenceTypes &...iOpSequences) {
executeOperationsFused(stream, fuseBackSequence(iOpSequences)...);
}

public:
FK_STATIC_STRUCT(Executor, SelfType)
FK_HOST_FUSE ParArch parArch() { return ParArch::CPU; }
template <typename... IOpSequenceTypes>
FK_HOST_FUSE void executeOperations(Stream_<ParArch::CPU> &stream,
const IOpSequenceTypes &...iOpSequences) {
executeOperations_helper(stream, iOpSequences...);
}
};

#if defined(__NVCC__)
struct ComputeBestSolutionBase {
FK_HOST_FUSE uint computeDiscardedThreads(const uint width, const uint height, const uint blockDimx, const uint blockDimy) {
Expand Down
91 changes: 91 additions & 0 deletions tests/data/test_circular_tensor.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/* Copyright 2026 Oscar Amoros Huguet

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */

#include "tests/main.h"

#include <fused_kernel/algorithms/basic_ops/memory_operations.h>
#include <fused_kernel/algorithms/image_processing/saturate.h>
#include <fused_kernel/core/data/circular_tensor.h>
#include <fused_kernel/core/data/ptr_utils.h>
#include <fused_kernel/fused_kernel.h>

#include <iostream>

template <uint BATCH, uint WIDTH, uint HEIGHT, uint ITERS, typename IT, typename OT>
bool testCircularTensor() {
using TensorOT = typename fk::VectorTraits<OT>::base;
constexpr uint COLOR_PLANES = fk::cn<IT>;

fk::CircularTensor<TensorOT, COLOR_PLANES, BATCH, fk::CircularTensorOrder::NewestFirst, fk::ColorPlanes::Standard>
myTensor(WIDTH, HEIGHT);
fk::Ptr2D<IT> input(WIDTH, HEIGHT);

fk::Stream fk_stream;
fk::setTo(10.0f, myTensor, fk_stream);

for (int i = 0; i < ITERS; i++) {
fk::setTo(fk::make_<IT>(i + 1, i + 1, i + 1), input, fk_stream);
myTensor.update(fk_stream, fk::Read<fk::PerThreadRead<fk::ND::_2D, IT>>{input.ptr()},
fk::Unary<fk::SaturateCast<IT, OT>>{}, fk::Write<fk::TensorSplit<OT>>{myTensor.ptr()});
fk_stream.sync();
}

myTensor.download(fk_stream);
fk_stream.sync();

bool correct = true;
for (int z = 0; z < BATCH; z++) {
const TensorOT value = (TensorOT)(ITERS - z);
for (int y = 0; y < HEIGHT; y++) {
for (int x = 0; x < WIDTH; x++) {
const fk::Point p{x, y, z};
const TensorOT res = *fk::PtrAccessor<fk::ND::_3D>::point(p, myTensor.ptrPinned());
correct &= value == res;
}
}
}

return correct;
}

template <uint BATCH, uint WIDTH, uint HEIGHT, uint ITERS, typename IT, typename OT>
bool launchTest() {
if (testCircularTensor<BATCH, WIDTH, HEIGHT, ITERS, IT, OT>()) {
std::cout << "testCircularTensor<" << BATCH << ", " << WIDTH << ", " << HEIGHT << ", " << ITERS << ", " << typeid(IT).name() << ", " << typeid(OT).name() << "> OK" << std::endl;
return true;
} else {
std::cout << "testCircularTensor<" << BATCH << ", " << WIDTH << ", " << HEIGHT << ", " << ITERS << ", " << typeid(IT).name() << ", " << typeid(OT).name() << "> Failed!"
<< std::endl;
return false;
}
}

int launch() {
bool correct = true;
correct &= launchTest<2, 128, 128, 100, uchar3, float3>();
correct &= launchTest<3, 128, 128, 100, uchar3, float3>();
correct &= launchTest<4, 128, 128, 100, uchar3, float3>();
correct &= launchTest<5, 128, 128, 100, uchar3, float3>();
correct &= launchTest<6, 128, 128, 100, uchar3, float3>();
correct &= launchTest<7, 128, 128, 100, uchar3, float3>();
correct &= launchTest<8, 128, 128, 100, uchar3, float3>();
correct &= launchTest<9, 128, 128, 100, uchar3, float3>();
correct &= launchTest<10, 128, 128, 100, uchar3, float3>();
correct &= launchTest<11, 128, 128, 100, uchar3, float3>();
correct &= launchTest<12, 128, 128, 100, uchar3, float3>();
correct &= launchTest<13, 128, 128, 100, uchar3, float3>();
correct &= launchTest<14, 128, 128, 100, uchar3, float3>();
correct &= launchTest<15, 128, 128, 100, uchar3, float3>();
return correct ? 0 : -1;
}
Loading
Loading