Skip to content

Fix make_buffer_copy unfoldable gap after skipped extent-1 dimensions #9681

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
derek-gerstmann wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix make_buffer_copy unfoldable gap after skipped extent-1 dimensions #9681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions src/runtime/device_buffer_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,31 +130,34 @@ WEAK device_copy make_buffer_copy(const halide_buffer_t *src, bool src_host,
// the extents and strides from the halide_buffer_ts. Dimensions
// are added to the copy by inserting it such that the stride is
// in ascending order in the dst.
int inserted = 0;
for (int i = 0; i < dst->dimensions; i++) {
int copy_extent = min(src->dim[i].extent, dst->dim[i].extent);
// Skip extent-one dimensions
if (dst->dim[i].extent == 1) {
if (copy_extent == 1) {
continue;
}
// TODO: deal with negative strides.
uint64_t dst_stride_bytes = (uint64_t)dst->dim[i].stride * dst->type.bytes();
uint64_t src_stride_bytes = (uint64_t)src->dim[i].stride * src->type.bytes();
// Insert the dimension sorted into the buffer copy.
int insert;
for (insert = 0; insert < i; insert++) {
for (insert = 0; insert < inserted; insert++) {
// If the stride is 0, we put it at the end because it can't be
// folded.
if (dst_stride_bytes < c.dst_stride_bytes[insert] && dst_stride_bytes != 0) {
break;
}
}
for (int j = i; j > insert; j--) {
for (int j = inserted; j > insert; j--) {
c.extent[j] = c.extent[j - 1];
c.dst_stride_bytes[j] = c.dst_stride_bytes[j - 1];
c.src_stride_bytes[j] = c.src_stride_bytes[j - 1];
}
c.extent[insert] = min(src->dim[i].extent, dst->dim[i].extent);
c.extent[insert] = copy_extent;
c.dst_stride_bytes[insert] = dst_stride_bytes;
c.src_stride_bytes[insert] = src_stride_bytes;
inserted++;
};

// Attempt to fold contiguous dimensions into the chunk
Expand Down
12 changes: 7 additions & 5 deletions test/performance/device_copy.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,15 @@ int main() {

double t1 = test(target, {3, 2, 0, 1}, {1024, 1024, 3, 2});
double t2 = test(target, {3, 2, 0, 1}, {1024, 1024, 6, 1});
double t3 = test(target, {0, 1, 2, 3}, {1024, 1024, 3, 2});
double t4 = test(target, {0, 1, 2, 3}, {1024, 1024, 6, 1});
double t3 = test(target, {3, 2, 0, 1}, {1, 1024, 1024, 6});
double t4 = test(target, {0, 1, 2, 3}, {1024, 1024, 3, 2});
double t5 = test(target, {0, 1, 2, 3}, {1024, 1024, 6, 1});
double t6 = test(target, {0, 1, 2, 3}, {1, 1024, 1024, 6});

double slowest = std::max({t1, t2, t3, t4});
double fastest = std::min({t1, t2, t3, t4});
double slowest = std::max({t1, t2, t3, t4, t5, t6});
double fastest = std::min({t1, t2, t3, t4, t5, t6});

printf("Timings: %f %f %f %f\n", t1, t2, t3, t4);
printf("Timings: %f %f %f %f %f %f\n", t1, t2, t3, t4, t5, t6);

// If one of these gets broken into a large number of separate copies, it
// will be a lot more than 10x slower.
Expand Down
100 changes: 99 additions & 1 deletion test/runtime/device_buffer_utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,105 @@ int main(int argc, char **argv) {
halide_buffer_t dst = make_buffer(u8, host, 2, dst_dim);

device_copy c = make_buffer_copy(&src, true, &dst, true);
HALIDE_CHECK(user_context, c.chunk_size != 0);
HALIDE_CHECK(user_context, c.chunk_size == 12);
for (int i = 0; i < MAX_COPY_DIMS; i++) {
HALIDE_CHECK(user_context, c.extent[i] == 1);
}
}

// Dense buffer with a leading extent-1 dimension (e.g. width-1 image)
// must fold into a single contiguous chunk (#9563).
{
halide_dimension_t dim[2] = {{0, 1, 1}, {0, 64, 1}};
uint8_t host[64] = {0};
halide_buffer_t src = make_buffer(u8, host, 2, dim);
halide_buffer_t dst = make_buffer(u8, host, 2, dim);

device_copy c = make_buffer_copy(&src, true, &dst, true);
HALIDE_CHECK(user_context, c.chunk_size == 64);
for (int i = 0; i < MAX_COPY_DIMS; i++) {
HALIDE_CHECK(user_context, c.extent[i] == 1);
HALIDE_CHECK(user_context, c.src_stride_bytes[i] == 0);
HALIDE_CHECK(user_context, c.dst_stride_bytes[i] == 0);
}
}

// Dense buffer with an interior extent-1 dimension (e.g. single-row planar
// {x: 64, y: 1, c: 3}) must fold into a single contiguous chunk (#9563).
{
halide_dimension_t dim[3] = {{0, 64, 1}, {0, 1, 64}, {0, 3, 64}};
uint8_t host[192] = {0};
halide_buffer_t src = make_buffer(u8, host, 3, dim);
halide_buffer_t dst = make_buffer(u8, host, 3, dim);

device_copy c = make_buffer_copy(&src, true, &dst, true);
HALIDE_CHECK(user_context, c.chunk_size == 192);
for (int i = 0; i < MAX_COPY_DIMS; i++) {
HALIDE_CHECK(user_context, c.extent[i] == 1);
HALIDE_CHECK(user_context, c.src_stride_bytes[i] == 0);
HALIDE_CHECK(user_context, c.dst_stride_bytes[i] == 0);
}
}

// Dense buffer with transposed strides and multiple extent-1 dimensions
// (including arbitrary strides on the extent-1 dimensions) must sort and
// fold into a single contiguous chunk.
{
halide_dimension_t dim[4] = {
{0, 1, 999},
{0, 5, 4},
{0, 1, 777},
{0, 4, 1},
};
uint8_t host[20] = {0};
halide_buffer_t src = make_buffer(u8, host, 4, dim);
halide_buffer_t dst = make_buffer(u8, host, 4, dim);

device_copy c = make_buffer_copy(&src, true, &dst, true);
HALIDE_CHECK(user_context, c.chunk_size == 20);
for (int i = 0; i < MAX_COPY_DIMS; i++) {
HALIDE_CHECK(user_context, c.extent[i] == 1);
}
}

// Strided (non-dense) copy with a skipped extent-1 dimension must fold
// the contiguous inner dimension and place the remaining outer dimension
// at index 0 with no unused gap slots before it.
{
halide_dimension_t dim[3] = {
{0, 1, 1},
{0, 8, 1},
{0, 4, 16},
};
uint8_t host[64] = {0};
halide_buffer_t src = make_buffer(u8, host, 3, dim);
halide_buffer_t dst = make_buffer(u8, host, 3, dim);

device_copy c = make_buffer_copy(&src, true, &dst, true);
HALIDE_CHECK(user_context, c.chunk_size == 8);
HALIDE_CHECK(user_context, c.extent[0] == 4);
HALIDE_CHECK(user_context, c.src_stride_bytes[0] == 16);
HALIDE_CHECK(user_context, c.dst_stride_bytes[0] == 16);
for (int i = 1; i < MAX_COPY_DIMS; i++) {
HALIDE_CHECK(user_context, c.extent[i] == 1);
}
}

// Overlapping crop copy where src has extent 1 in a dimension where dst
// has extent > 1 should also skip the extent-1 copy dimension and fold.
{
halide_dimension_t src_dim[3] = {{0, 16, 1}, {0, 1, 999}, {0, 3, 16}};
halide_dimension_t dst_dim[3] = {{0, 16, 1}, {0, 10, 48}, {0, 3, 16}};
uint8_t src_host[48] = {0};
uint8_t dst_host[480] = {0};
halide_buffer_t src = make_buffer(u8, src_host, 3, src_dim);
halide_buffer_t dst = make_buffer(u8, dst_host, 3, dst_dim);

device_copy c = make_buffer_copy(&src, true, &dst, true);
HALIDE_CHECK(user_context, c.chunk_size == 48);
for (int i = 0; i < MAX_COPY_DIMS; i++) {
HALIDE_CHECK(user_context, c.extent[i] == 1);
}
}

print(user_context) << "Success!\n";
Expand Down
Loading