wgpu_core/command/
transfer.rs

1use alloc::{sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{BufferAddress, BufferUsages, Extent3d, TextureSelector, TextureUsages};
6
7#[cfg(feature = "trace")]
8use crate::device::trace::Command as TraceCommand;
9use crate::{
10    api_log,
11    command::{clear_texture, CommandEncoderError},
12    conv,
13    device::{Device, DeviceError, MissingDownlevelFlags},
14    global::Global,
15    id::{BufferId, CommandEncoderId, TextureId},
16    init_tracker::{
17        has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
18        TextureInitTrackerAction,
19    },
20    resource::{
21        DestroyedResourceError, InvalidResourceError, MissingBufferUsageError,
22        MissingTextureUsageError, ParentDevice, Texture, TextureErrorDimension,
23    },
24    snatch::SnatchGuard,
25};
26
27use super::{ClearError, CommandBufferMutable};
28
29pub type TexelCopyBufferInfo = wgt::TexelCopyBufferInfo<BufferId>;
30pub type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<TextureId>;
31pub type CopyExternalImageDestInfo = wgt::CopyExternalImageDestInfo<TextureId>;
32
33#[deprecated(
34    since = "24.0.0",
35    note = "This has been renamed to `TexelCopyBufferInfo`, and will be removed in 25.0.0."
36)]
37pub type ImageCopyBuffer = wgt::TexelCopyBufferInfo<BufferId>;
38
39#[deprecated(
40    since = "24.0.0",
41    note = "This has been renamed to `TexelCopyTextureInfo`, and will be removed in 25.0.0."
42)]
43pub type ImageCopyTexture = wgt::TexelCopyTextureInfo<TextureId>;
44
45#[deprecated(
46    since = "24.0.0",
47    note = "This has been renamed to `TexelCopyTextureSourceInfo`, and will be removed in 25.0.0."
48)]
49pub type ImageCopyTextureTagged = wgt::CopyExternalImageDestInfo<TextureId>;
50
51#[derive(Clone, Copy, Debug)]
52pub enum CopySide {
53    Source,
54    Destination,
55}
56
57/// Error encountered while attempting a data transfer.
58#[derive(Clone, Debug, Error)]
59#[non_exhaustive]
60pub enum TransferError {
61    #[error("Source and destination cannot be the same buffer")]
62    SameSourceDestinationBuffer,
63    #[error(transparent)]
64    MissingBufferUsage(#[from] MissingBufferUsageError),
65    #[error(transparent)]
66    MissingTextureUsage(#[from] MissingTextureUsageError),
67    #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
68    BufferOverrun {
69        start_offset: BufferAddress,
70        end_offset: BufferAddress,
71        buffer_size: BufferAddress,
72        side: CopySide,
73    },
74    #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
75    TextureOverrun {
76        start_offset: u32,
77        end_offset: u32,
78        texture_size: u32,
79        dimension: TextureErrorDimension,
80        side: CopySide,
81    },
82    #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
83    InvalidTextureAspect {
84        format: wgt::TextureFormat,
85        aspect: wgt::TextureAspect,
86    },
87    #[error("Unable to select texture mip level {level} out of {total}")]
88    InvalidTextureMipLevel { level: u32, total: u32 },
89    #[error("Texture dimension must be 2D when copying from an external texture")]
90    InvalidDimensionExternal,
91    #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
92    UnalignedBufferOffset(BufferAddress),
93    #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
94    UnalignedCopySize(BufferAddress),
95    #[error("Copy width is not a multiple of block width")]
96    UnalignedCopyWidth,
97    #[error("Copy height is not a multiple of block height")]
98    UnalignedCopyHeight,
99    #[error("Copy origin's x component is not a multiple of block width")]
100    UnalignedCopyOriginX,
101    #[error("Copy origin's y component is not a multiple of block height")]
102    UnalignedCopyOriginY,
103    #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
104    UnalignedBytesPerRow,
105    #[error("Number of bytes per row needs to be specified since more than one row is copied")]
106    UnspecifiedBytesPerRow,
107    #[error("Number of rows per image needs to be specified since more than one image is copied")]
108    UnspecifiedRowsPerImage,
109    #[error("Number of bytes per row is less than the number of bytes in a complete row")]
110    InvalidBytesPerRow,
111    #[error("Number of rows per image is invalid")]
112    InvalidRowsPerImage,
113    #[error("Copy source aspects must refer to all aspects of the source texture format")]
114    CopySrcMissingAspects,
115    #[error(
116        "Copy destination aspects must refer to all aspects of the destination texture format"
117    )]
118    CopyDstMissingAspects,
119    #[error("Copy aspect must refer to a single aspect of texture format")]
120    CopyAspectNotOne,
121    #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
122    CopyFromForbiddenTextureFormat {
123        format: wgt::TextureFormat,
124        aspect: wgt::TextureAspect,
125    },
126    #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
127    CopyToForbiddenTextureFormat {
128        format: wgt::TextureFormat,
129        aspect: wgt::TextureAspect,
130    },
131    #[error(
132        "Copying to textures with format {0:?} is forbidden when copying from external texture"
133    )]
134    ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
135    #[error(
136        "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
137    )]
138    TextureFormatsNotCopyCompatible {
139        src_format: wgt::TextureFormat,
140        dst_format: wgt::TextureFormat,
141    },
142    #[error(transparent)]
143    MemoryInitFailure(#[from] ClearError),
144    #[error("Cannot encode this copy because of a missing downelevel flag")]
145    MissingDownlevelFlags(#[from] MissingDownlevelFlags),
146    #[error("Source texture sample count must be 1, got {sample_count}")]
147    InvalidSampleCount { sample_count: u32 },
148    #[error(
149        "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
150    )]
151    SampleCountNotEqual {
152        src_sample_count: u32,
153        dst_sample_count: u32,
154    },
155    #[error("Requested mip level {requested} does no exist (count: {count})")]
156    InvalidMipLevel { requested: u32, count: u32 },
157}
158
159/// Error encountered while attempting to do a copy on a command encoder.
160#[derive(Clone, Debug, Error)]
161#[non_exhaustive]
162pub enum CopyError {
163    #[error(transparent)]
164    Encoder(#[from] CommandEncoderError),
165    #[error("Copy error")]
166    Transfer(#[from] TransferError),
167    #[error(transparent)]
168    DestroyedResource(#[from] DestroyedResourceError),
169    #[error(transparent)]
170    InvalidResource(#[from] InvalidResourceError),
171}
172
173impl From<DeviceError> for CopyError {
174    fn from(err: DeviceError) -> Self {
175        CopyError::Encoder(CommandEncoderError::Device(err))
176    }
177}
178
179pub(crate) fn extract_texture_selector<T>(
180    copy_texture: &wgt::TexelCopyTextureInfo<T>,
181    copy_size: &Extent3d,
182    texture: &Texture,
183) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
184    let format = texture.desc.format;
185    let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
186    if copy_aspect.is_empty() {
187        return Err(TransferError::InvalidTextureAspect {
188            format,
189            aspect: copy_texture.aspect,
190        });
191    }
192
193    let (layers, origin_z) = match texture.desc.dimension {
194        wgt::TextureDimension::D1 => (0..1, 0),
195        wgt::TextureDimension::D2 => (
196            copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
197            0,
198        ),
199        wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
200    };
201    let base = hal::TextureCopyBase {
202        origin: wgt::Origin3d {
203            x: copy_texture.origin.x,
204            y: copy_texture.origin.y,
205            z: origin_z,
206        },
207        // this value will be incremented per copied layer
208        array_layer: layers.start,
209        mip_level: copy_texture.mip_level,
210        aspect: copy_aspect,
211    };
212    let selector = TextureSelector {
213        mips: copy_texture.mip_level..copy_texture.mip_level + 1,
214        layers,
215    };
216
217    Ok((selector, base))
218}
219
220/// WebGPU's [validating linear texture data][vltd] algorithm.
221///
222/// Copied with some modifications from WebGPU standard.
223///
224/// If successful, returns a pair `(bytes, stride)`, where:
225/// - `bytes` is the number of buffer bytes required for this copy, and
226/// - `stride` number of bytes between array layers.
227///
228/// [vltd]: https://gpuweb.github.io/gpuweb/#abstract-opdef-validating-linear-texture-data
229pub(crate) fn validate_linear_texture_data(
230    layout: &wgt::TexelCopyBufferLayout,
231    format: wgt::TextureFormat,
232    aspect: wgt::TextureAspect,
233    buffer_size: BufferAddress,
234    buffer_side: CopySide,
235    copy_size: &Extent3d,
236    need_copy_aligned_rows: bool,
237) -> Result<(BufferAddress, BufferAddress), TransferError> {
238    // Convert all inputs to BufferAddress (u64) to avoid some of the overflow issues
239    // Note: u64 is not always enough to prevent overflow, especially when multiplying
240    // something with a potentially large depth value, so it is preferable to validate
241    // the copy size before calling this function (for example via `validate_texture_copy_range`).
242    let copy_width = copy_size.width as BufferAddress;
243    let copy_height = copy_size.height as BufferAddress;
244    let depth_or_array_layers = copy_size.depth_or_array_layers as BufferAddress;
245
246    let offset = layout.offset;
247
248    let block_size = format.block_copy_size(Some(aspect)).unwrap() as BufferAddress;
249    let (block_width, block_height) = format.block_dimensions();
250    let block_width = block_width as BufferAddress;
251    let block_height = block_height as BufferAddress;
252
253    if copy_width % block_width != 0 {
254        return Err(TransferError::UnalignedCopyWidth);
255    }
256    if copy_height % block_height != 0 {
257        return Err(TransferError::UnalignedCopyHeight);
258    }
259
260    let width_in_blocks = copy_width / block_width;
261    let height_in_blocks = copy_height / block_height;
262
263    let bytes_in_last_row = width_in_blocks * block_size;
264
265    let bytes_per_row = if let Some(bytes_per_row) = layout.bytes_per_row {
266        let bytes_per_row = bytes_per_row as BufferAddress;
267        if bytes_per_row < bytes_in_last_row {
268            return Err(TransferError::InvalidBytesPerRow);
269        }
270        bytes_per_row
271    } else {
272        if depth_or_array_layers > 1 || height_in_blocks > 1 {
273            return Err(TransferError::UnspecifiedBytesPerRow);
274        }
275        0
276    };
277    let rows_per_image = if let Some(rows_per_image) = layout.rows_per_image {
278        let rows_per_image = rows_per_image as BufferAddress;
279        if rows_per_image < height_in_blocks {
280            return Err(TransferError::InvalidRowsPerImage);
281        }
282        rows_per_image
283    } else {
284        if depth_or_array_layers > 1 {
285            return Err(TransferError::UnspecifiedRowsPerImage);
286        }
287        0
288    };
289
290    if need_copy_aligned_rows {
291        let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
292
293        let mut offset_alignment = block_size;
294        if format.is_depth_stencil_format() {
295            offset_alignment = 4
296        }
297        if offset % offset_alignment != 0 {
298            return Err(TransferError::UnalignedBufferOffset(offset));
299        }
300
301        if bytes_per_row % bytes_per_row_alignment != 0 {
302            return Err(TransferError::UnalignedBytesPerRow);
303        }
304    }
305
306    let bytes_per_image = bytes_per_row * rows_per_image;
307
308    let required_bytes_in_copy = if depth_or_array_layers == 0 {
309        0
310    } else {
311        let mut required_bytes_in_copy = bytes_per_image * (depth_or_array_layers - 1);
312        if height_in_blocks > 0 {
313            required_bytes_in_copy += bytes_per_row * (height_in_blocks - 1) + bytes_in_last_row;
314        }
315        required_bytes_in_copy
316    };
317
318    if offset + required_bytes_in_copy > buffer_size {
319        return Err(TransferError::BufferOverrun {
320            start_offset: offset,
321            end_offset: offset + required_bytes_in_copy,
322            buffer_size,
323            side: buffer_side,
324        });
325    }
326
327    Ok((required_bytes_in_copy, bytes_per_image))
328}
329
330/// WebGPU's [validating texture copy range][vtcr] algorithm.
331///
332/// Copied with minor modifications from WebGPU standard.
333///
334/// Returns the HAL copy extent and the layer count.
335///
336/// [vtcr]: https://gpuweb.github.io/gpuweb/#validating-texture-copy-range
337pub(crate) fn validate_texture_copy_range<T>(
338    texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
339    desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
340    texture_side: CopySide,
341    copy_size: &Extent3d,
342) -> Result<(hal::CopyExtent, u32), TransferError> {
343    let (block_width, block_height) = desc.format.block_dimensions();
344
345    let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
346        TransferError::InvalidTextureMipLevel {
347            level: texture_copy_view.mip_level,
348            total: desc.mip_level_count,
349        },
350    )?;
351    // physical size can be larger than the virtual
352    let extent = extent_virtual.physical_size(desc.format);
353
354    /// Return `Ok` if a run `size` texels long starting at `start_offset` falls
355    /// entirely within `texture_size`. Otherwise, return an appropriate a`Err`.
356    fn check_dimension(
357        dimension: TextureErrorDimension,
358        side: CopySide,
359        start_offset: u32,
360        size: u32,
361        texture_size: u32,
362    ) -> Result<(), TransferError> {
363        // Avoid underflow in the subtraction by checking start_offset against
364        // texture_size first.
365        if start_offset <= texture_size && size <= texture_size - start_offset {
366            Ok(())
367        } else {
368            Err(TransferError::TextureOverrun {
369                start_offset,
370                end_offset: start_offset.wrapping_add(size),
371                texture_size,
372                dimension,
373                side,
374            })
375        }
376    }
377
378    check_dimension(
379        TextureErrorDimension::X,
380        texture_side,
381        texture_copy_view.origin.x,
382        copy_size.width,
383        extent.width,
384    )?;
385    check_dimension(
386        TextureErrorDimension::Y,
387        texture_side,
388        texture_copy_view.origin.y,
389        copy_size.height,
390        extent.height,
391    )?;
392    check_dimension(
393        TextureErrorDimension::Z,
394        texture_side,
395        texture_copy_view.origin.z,
396        copy_size.depth_or_array_layers,
397        extent.depth_or_array_layers,
398    )?;
399
400    if texture_copy_view.origin.x % block_width != 0 {
401        return Err(TransferError::UnalignedCopyOriginX);
402    }
403    if texture_copy_view.origin.y % block_height != 0 {
404        return Err(TransferError::UnalignedCopyOriginY);
405    }
406    if copy_size.width % block_width != 0 {
407        return Err(TransferError::UnalignedCopyWidth);
408    }
409    if copy_size.height % block_height != 0 {
410        return Err(TransferError::UnalignedCopyHeight);
411    }
412
413    let (depth, array_layer_count) = match desc.dimension {
414        wgt::TextureDimension::D1 => (1, 1),
415        wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
416        wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
417    };
418
419    let copy_extent = hal::CopyExtent {
420        width: copy_size.width,
421        height: copy_size.height,
422        depth,
423    };
424    Ok((copy_extent, array_layer_count))
425}
426
427fn handle_texture_init(
428    init_kind: MemoryInitKind,
429    cmd_buf_data: &mut CommandBufferMutable,
430    device: &Device,
431    copy_texture: &TexelCopyTextureInfo,
432    copy_size: &Extent3d,
433    texture: &Arc<Texture>,
434    snatch_guard: &SnatchGuard<'_>,
435) -> Result<(), ClearError> {
436    let init_action = TextureInitTrackerAction {
437        texture: texture.clone(),
438        range: TextureInitRange {
439            mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
440            layer_range: copy_texture.origin.z
441                ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
442        },
443        kind: init_kind,
444    };
445
446    // Register the init action.
447    let immediate_inits = cmd_buf_data
448        .texture_memory_actions
449        .register_init_action(&{ init_action });
450
451    // In rare cases we may need to insert an init operation immediately onto the command buffer.
452    if !immediate_inits.is_empty() {
453        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
454        for init in immediate_inits {
455            clear_texture(
456                &init.texture,
457                TextureInitRange {
458                    mip_range: init.mip_level..(init.mip_level + 1),
459                    layer_range: init.layer..(init.layer + 1),
460                },
461                cmd_buf_raw,
462                &mut cmd_buf_data.trackers.textures,
463                &device.alignments,
464                device.zero_buffer.as_ref(),
465                snatch_guard,
466            )?;
467        }
468    }
469
470    Ok(())
471}
472
473/// Prepare a transfer's source texture.
474///
475/// Ensure the source texture of a transfer is in the right initialization
476/// state, and record the state for after the transfer operation.
477fn handle_src_texture_init(
478    cmd_buf_data: &mut CommandBufferMutable,
479    device: &Device,
480    source: &TexelCopyTextureInfo,
481    copy_size: &Extent3d,
482    texture: &Arc<Texture>,
483    snatch_guard: &SnatchGuard<'_>,
484) -> Result<(), TransferError> {
485    handle_texture_init(
486        MemoryInitKind::NeedsInitializedMemory,
487        cmd_buf_data,
488        device,
489        source,
490        copy_size,
491        texture,
492        snatch_guard,
493    )?;
494    Ok(())
495}
496
497/// Prepare a transfer's destination texture.
498///
499/// Ensure the destination texture of a transfer is in the right initialization
500/// state, and record the state for after the transfer operation.
501fn handle_dst_texture_init(
502    cmd_buf_data: &mut CommandBufferMutable,
503    device: &Device,
504    destination: &TexelCopyTextureInfo,
505    copy_size: &Extent3d,
506    texture: &Arc<Texture>,
507    snatch_guard: &SnatchGuard<'_>,
508) -> Result<(), TransferError> {
509    // Attention: If we don't write full texture subresources, we need to a full
510    // clear first since we don't track subrects. This means that in rare cases
511    // even a *destination* texture of a transfer may need an immediate texture
512    // init.
513    let dst_init_kind = if has_copy_partial_init_tracker_coverage(
514        copy_size,
515        destination.mip_level,
516        &texture.desc,
517    ) {
518        MemoryInitKind::NeedsInitializedMemory
519    } else {
520        MemoryInitKind::ImplicitlyInitialized
521    };
522
523    handle_texture_init(
524        dst_init_kind,
525        cmd_buf_data,
526        device,
527        destination,
528        copy_size,
529        texture,
530        snatch_guard,
531    )?;
532    Ok(())
533}
534
535impl Global {
536    pub fn command_encoder_copy_buffer_to_buffer(
537        &self,
538        command_encoder_id: CommandEncoderId,
539        source: BufferId,
540        source_offset: BufferAddress,
541        destination: BufferId,
542        destination_offset: BufferAddress,
543        size: BufferAddress,
544    ) -> Result<(), CopyError> {
545        profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
546        api_log!(
547            "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
548        );
549
550        if source == destination {
551            return Err(TransferError::SameSourceDestinationBuffer.into());
552        }
553        let hub = &self.hub;
554
555        let cmd_buf = hub
556            .command_buffers
557            .get(command_encoder_id.into_command_buffer_id());
558        let mut cmd_buf_data = cmd_buf.data.lock();
559        let mut cmd_buf_data_guard = cmd_buf_data.record()?;
560        let cmd_buf_data = &mut *cmd_buf_data_guard;
561
562        let device = &cmd_buf.device;
563        device.check_is_valid()?;
564
565        #[cfg(feature = "trace")]
566        if let Some(ref mut list) = cmd_buf_data.commands {
567            list.push(TraceCommand::CopyBufferToBuffer {
568                src: source,
569                src_offset: source_offset,
570                dst: destination,
571                dst_offset: destination_offset,
572                size,
573            });
574        }
575
576        let snatch_guard = device.snatchable_lock.read();
577
578        let src_buffer = hub.buffers.get(source).get()?;
579
580        src_buffer.same_device_as(cmd_buf.as_ref())?;
581
582        let src_pending = cmd_buf_data
583            .trackers
584            .buffers
585            .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
586
587        let src_raw = src_buffer.try_raw(&snatch_guard)?;
588        src_buffer
589            .check_usage(BufferUsages::COPY_SRC)
590            .map_err(TransferError::MissingBufferUsage)?;
591        // expecting only a single barrier
592        let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
593
594        let dst_buffer = hub.buffers.get(destination).get()?;
595
596        dst_buffer.same_device_as(cmd_buf.as_ref())?;
597
598        let dst_pending = cmd_buf_data
599            .trackers
600            .buffers
601            .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
602
603        let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
604        dst_buffer
605            .check_usage(BufferUsages::COPY_DST)
606            .map_err(TransferError::MissingBufferUsage)?;
607        let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
608
609        if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
610            return Err(TransferError::UnalignedCopySize(size).into());
611        }
612        if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
613            return Err(TransferError::UnalignedBufferOffset(source_offset).into());
614        }
615        if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
616            return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
617        }
618        if !device
619            .downlevel
620            .flags
621            .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
622            && (src_buffer.usage.contains(BufferUsages::INDEX)
623                || dst_buffer.usage.contains(BufferUsages::INDEX))
624        {
625            let forbidden_usages = BufferUsages::VERTEX
626                | BufferUsages::UNIFORM
627                | BufferUsages::INDIRECT
628                | BufferUsages::STORAGE;
629            if src_buffer.usage.intersects(forbidden_usages)
630                || dst_buffer.usage.intersects(forbidden_usages)
631            {
632                return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
633                    wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
634                ))
635                .into());
636            }
637        }
638
639        let source_end_offset = source_offset + size;
640        let destination_end_offset = destination_offset + size;
641        if source_end_offset > src_buffer.size {
642            return Err(TransferError::BufferOverrun {
643                start_offset: source_offset,
644                end_offset: source_end_offset,
645                buffer_size: src_buffer.size,
646                side: CopySide::Source,
647            }
648            .into());
649        }
650        if destination_end_offset > dst_buffer.size {
651            return Err(TransferError::BufferOverrun {
652                start_offset: destination_offset,
653                end_offset: destination_end_offset,
654                buffer_size: dst_buffer.size,
655                side: CopySide::Destination,
656            }
657            .into());
658        }
659
660        if size == 0 {
661            log::trace!("Ignoring copy_buffer_to_buffer of size 0");
662            cmd_buf_data_guard.mark_successful();
663            return Ok(());
664        }
665
666        // Make sure source is initialized memory and mark dest as initialized.
667        cmd_buf_data.buffer_memory_init_actions.extend(
668            dst_buffer.initialization_status.read().create_action(
669                &dst_buffer,
670                destination_offset..(destination_offset + size),
671                MemoryInitKind::ImplicitlyInitialized,
672            ),
673        );
674        cmd_buf_data.buffer_memory_init_actions.extend(
675            src_buffer.initialization_status.read().create_action(
676                &src_buffer,
677                source_offset..(source_offset + size),
678                MemoryInitKind::NeedsInitializedMemory,
679            ),
680        );
681
682        let region = hal::BufferCopy {
683            src_offset: source_offset,
684            dst_offset: destination_offset,
685            size: wgt::BufferSize::new(size).unwrap(),
686        };
687        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
688        let barriers = src_barrier
689            .into_iter()
690            .chain(dst_barrier)
691            .collect::<Vec<_>>();
692        unsafe {
693            cmd_buf_raw.transition_buffers(&barriers);
694            cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
695        }
696
697        cmd_buf_data_guard.mark_successful();
698        Ok(())
699    }
700
701    pub fn command_encoder_copy_buffer_to_texture(
702        &self,
703        command_encoder_id: CommandEncoderId,
704        source: &TexelCopyBufferInfo,
705        destination: &TexelCopyTextureInfo,
706        copy_size: &Extent3d,
707    ) -> Result<(), CopyError> {
708        profiling::scope!("CommandEncoder::copy_buffer_to_texture");
709        api_log!(
710            "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
711            source.buffer,
712            destination.texture
713        );
714
715        let hub = &self.hub;
716
717        let cmd_buf = hub
718            .command_buffers
719            .get(command_encoder_id.into_command_buffer_id());
720        let mut cmd_buf_data = cmd_buf.data.lock();
721        let mut cmd_buf_data_guard = cmd_buf_data.record()?;
722        let cmd_buf_data = &mut *cmd_buf_data_guard;
723
724        let device = &cmd_buf.device;
725        device.check_is_valid()?;
726
727        #[cfg(feature = "trace")]
728        if let Some(ref mut list) = cmd_buf_data.commands {
729            list.push(TraceCommand::CopyBufferToTexture {
730                src: *source,
731                dst: *destination,
732                size: *copy_size,
733            });
734        }
735
736        if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
737            log::trace!("Ignoring copy_buffer_to_texture of size 0");
738            cmd_buf_data_guard.mark_successful();
739            return Ok(());
740        }
741
742        let dst_texture = hub.textures.get(destination.texture).get()?;
743
744        dst_texture.same_device_as(cmd_buf.as_ref())?;
745
746        let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
747            destination,
748            &dst_texture.desc,
749            CopySide::Destination,
750            copy_size,
751        )?;
752
753        let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?;
754
755        let snatch_guard = device.snatchable_lock.read();
756
757        // Handle texture init *before* dealing with barrier transitions so we
758        // have an easier time inserting "immediate-inits" that may be required
759        // by prior discards in rare cases.
760        handle_dst_texture_init(
761            cmd_buf_data,
762            device,
763            destination,
764            copy_size,
765            &dst_texture,
766            &snatch_guard,
767        )?;
768
769        let src_buffer = hub.buffers.get(source.buffer).get()?;
770
771        src_buffer.same_device_as(cmd_buf.as_ref())?;
772
773        let src_pending = cmd_buf_data
774            .trackers
775            .buffers
776            .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
777
778        let src_raw = src_buffer.try_raw(&snatch_guard)?;
779        src_buffer
780            .check_usage(BufferUsages::COPY_SRC)
781            .map_err(TransferError::MissingBufferUsage)?;
782        let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
783
784        let dst_pending = cmd_buf_data.trackers.textures.set_single(
785            &dst_texture,
786            dst_range,
787            wgt::TextureUses::COPY_DST,
788        );
789        let dst_raw = dst_texture.try_raw(&snatch_guard)?;
790        dst_texture
791            .check_usage(TextureUsages::COPY_DST)
792            .map_err(TransferError::MissingTextureUsage)?;
793        let dst_barrier = dst_pending
794            .map(|pending| pending.into_hal(dst_raw))
795            .collect::<Vec<_>>();
796
797        if !dst_base.aspect.is_one() {
798            return Err(TransferError::CopyAspectNotOne.into());
799        }
800
801        if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format, destination.aspect) {
802            return Err(TransferError::CopyToForbiddenTextureFormat {
803                format: dst_texture.desc.format,
804                aspect: destination.aspect,
805            }
806            .into());
807        }
808
809        let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
810            &source.layout,
811            dst_texture.desc.format,
812            destination.aspect,
813            src_buffer.size,
814            CopySide::Source,
815            copy_size,
816            true,
817        )?;
818
819        if dst_texture.desc.format.is_depth_stencil_format() {
820            device
821                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
822                .map_err(TransferError::from)?;
823        }
824
825        cmd_buf_data.buffer_memory_init_actions.extend(
826            src_buffer.initialization_status.read().create_action(
827                &src_buffer,
828                source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
829                MemoryInitKind::NeedsInitializedMemory,
830            ),
831        );
832
833        let regions = (0..array_layer_count)
834            .map(|rel_array_layer| {
835                let mut texture_base = dst_base.clone();
836                texture_base.array_layer += rel_array_layer;
837                let mut buffer_layout = source.layout;
838                buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
839                hal::BufferTextureCopy {
840                    buffer_layout,
841                    texture_base,
842                    size: hal_copy_size,
843                }
844            })
845            .collect::<Vec<_>>();
846
847        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
848        unsafe {
849            cmd_buf_raw.transition_textures(&dst_barrier);
850            cmd_buf_raw.transition_buffers(src_barrier.as_slice());
851            cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, &regions);
852        }
853
854        cmd_buf_data_guard.mark_successful();
855        Ok(())
856    }
857
858    pub fn command_encoder_copy_texture_to_buffer(
859        &self,
860        command_encoder_id: CommandEncoderId,
861        source: &TexelCopyTextureInfo,
862        destination: &TexelCopyBufferInfo,
863        copy_size: &Extent3d,
864    ) -> Result<(), CopyError> {
865        profiling::scope!("CommandEncoder::copy_texture_to_buffer");
866        api_log!(
867            "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
868            source.texture,
869            destination.buffer
870        );
871
872        let hub = &self.hub;
873
874        let cmd_buf = hub
875            .command_buffers
876            .get(command_encoder_id.into_command_buffer_id());
877        let mut cmd_buf_data = cmd_buf.data.lock();
878        let mut cmd_buf_data_guard = cmd_buf_data.record()?;
879        let cmd_buf_data = &mut *cmd_buf_data_guard;
880
881        let device = &cmd_buf.device;
882        device.check_is_valid()?;
883
884        #[cfg(feature = "trace")]
885        if let Some(list) = cmd_buf_data.commands.as_mut() {
886            list.push(TraceCommand::CopyTextureToBuffer {
887                src: *source,
888                dst: *destination,
889                size: *copy_size,
890            });
891        }
892
893        if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
894            log::trace!("Ignoring copy_texture_to_buffer of size 0");
895            cmd_buf_data_guard.mark_successful();
896            return Ok(());
897        }
898
899        let src_texture = hub.textures.get(source.texture).get()?;
900
901        src_texture.same_device_as(cmd_buf.as_ref())?;
902
903        let (hal_copy_size, array_layer_count) =
904            validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
905
906        let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
907
908        let snatch_guard = device.snatchable_lock.read();
909
910        // Handle texture init *before* dealing with barrier transitions so we
911        // have an easier time inserting "immediate-inits" that may be required
912        // by prior discards in rare cases.
913        handle_src_texture_init(
914            cmd_buf_data,
915            device,
916            source,
917            copy_size,
918            &src_texture,
919            &snatch_guard,
920        )?;
921
922        let src_pending = cmd_buf_data.trackers.textures.set_single(
923            &src_texture,
924            src_range,
925            wgt::TextureUses::COPY_SRC,
926        );
927        let src_raw = src_texture.try_raw(&snatch_guard)?;
928        src_texture
929            .check_usage(TextureUsages::COPY_SRC)
930            .map_err(TransferError::MissingTextureUsage)?;
931        if src_texture.desc.sample_count != 1 {
932            return Err(TransferError::InvalidSampleCount {
933                sample_count: src_texture.desc.sample_count,
934            }
935            .into());
936        }
937        if source.mip_level >= src_texture.desc.mip_level_count {
938            return Err(TransferError::InvalidMipLevel {
939                requested: source.mip_level,
940                count: src_texture.desc.mip_level_count,
941            }
942            .into());
943        }
944        let src_barrier = src_pending
945            .map(|pending| pending.into_hal(src_raw))
946            .collect::<Vec<_>>();
947
948        let dst_buffer = hub.buffers.get(destination.buffer).get()?;
949
950        dst_buffer.same_device_as(cmd_buf.as_ref())?;
951
952        let dst_pending = cmd_buf_data
953            .trackers
954            .buffers
955            .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
956
957        let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
958        dst_buffer
959            .check_usage(BufferUsages::COPY_DST)
960            .map_err(TransferError::MissingBufferUsage)?;
961        let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
962
963        if !src_base.aspect.is_one() {
964            return Err(TransferError::CopyAspectNotOne.into());
965        }
966
967        if !conv::is_valid_copy_src_texture_format(src_texture.desc.format, source.aspect) {
968            return Err(TransferError::CopyFromForbiddenTextureFormat {
969                format: src_texture.desc.format,
970                aspect: source.aspect,
971            }
972            .into());
973        }
974
975        let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
976            &destination.layout,
977            src_texture.desc.format,
978            source.aspect,
979            dst_buffer.size,
980            CopySide::Destination,
981            copy_size,
982            true,
983        )?;
984
985        if src_texture.desc.format.is_depth_stencil_format() {
986            device
987                .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
988                .map_err(TransferError::from)?;
989        }
990
991        cmd_buf_data.buffer_memory_init_actions.extend(
992            dst_buffer.initialization_status.read().create_action(
993                &dst_buffer,
994                destination.layout.offset
995                    ..(destination.layout.offset + required_buffer_bytes_in_copy),
996                MemoryInitKind::ImplicitlyInitialized,
997            ),
998        );
999
1000        let regions = (0..array_layer_count)
1001            .map(|rel_array_layer| {
1002                let mut texture_base = src_base.clone();
1003                texture_base.array_layer += rel_array_layer;
1004                let mut buffer_layout = destination.layout;
1005                buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1006                hal::BufferTextureCopy {
1007                    buffer_layout,
1008                    texture_base,
1009                    size: hal_copy_size,
1010                }
1011            })
1012            .collect::<Vec<_>>();
1013        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1014        unsafe {
1015            cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
1016            cmd_buf_raw.transition_textures(&src_barrier);
1017            cmd_buf_raw.copy_texture_to_buffer(
1018                src_raw,
1019                wgt::TextureUses::COPY_SRC,
1020                dst_raw,
1021                &regions,
1022            );
1023        }
1024
1025        cmd_buf_data_guard.mark_successful();
1026        Ok(())
1027    }
1028
1029    pub fn command_encoder_copy_texture_to_texture(
1030        &self,
1031        command_encoder_id: CommandEncoderId,
1032        source: &TexelCopyTextureInfo,
1033        destination: &TexelCopyTextureInfo,
1034        copy_size: &Extent3d,
1035    ) -> Result<(), CopyError> {
1036        profiling::scope!("CommandEncoder::copy_texture_to_texture");
1037        api_log!(
1038            "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1039            source.texture,
1040            destination.texture
1041        );
1042
1043        let hub = &self.hub;
1044
1045        let cmd_buf = hub
1046            .command_buffers
1047            .get(command_encoder_id.into_command_buffer_id());
1048        let mut cmd_buf_data = cmd_buf.data.lock();
1049        let mut cmd_buf_data_guard = cmd_buf_data.record()?;
1050        let cmd_buf_data = &mut *cmd_buf_data_guard;
1051
1052        let device = &cmd_buf.device;
1053        device.check_is_valid()?;
1054
1055        let snatch_guard = device.snatchable_lock.read();
1056
1057        #[cfg(feature = "trace")]
1058        if let Some(ref mut list) = cmd_buf_data.commands {
1059            list.push(TraceCommand::CopyTextureToTexture {
1060                src: *source,
1061                dst: *destination,
1062                size: *copy_size,
1063            });
1064        }
1065
1066        if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1067            log::trace!("Ignoring copy_texture_to_texture of size 0");
1068            cmd_buf_data_guard.mark_successful();
1069            return Ok(());
1070        }
1071
1072        let src_texture = hub.textures.get(source.texture).get()?;
1073        let dst_texture = hub.textures.get(destination.texture).get()?;
1074
1075        src_texture.same_device_as(cmd_buf.as_ref())?;
1076        dst_texture.same_device_as(cmd_buf.as_ref())?;
1077
1078        // src and dst texture format must be copy-compatible
1079        // https://gpuweb.github.io/gpuweb/#copy-compatible
1080        if src_texture.desc.format.remove_srgb_suffix()
1081            != dst_texture.desc.format.remove_srgb_suffix()
1082        {
1083            return Err(TransferError::TextureFormatsNotCopyCompatible {
1084                src_format: src_texture.desc.format,
1085                dst_format: dst_texture.desc.format,
1086            }
1087            .into());
1088        }
1089
1090        let (src_copy_size, array_layer_count) =
1091            validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1092        let (dst_copy_size, _) = validate_texture_copy_range(
1093            destination,
1094            &dst_texture.desc,
1095            CopySide::Destination,
1096            copy_size,
1097        )?;
1098
1099        let (src_range, src_tex_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1100        let (dst_range, dst_tex_base) =
1101            extract_texture_selector(destination, copy_size, &dst_texture)?;
1102        let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1103        let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1104        if src_tex_base.aspect != src_texture_aspects {
1105            return Err(TransferError::CopySrcMissingAspects.into());
1106        }
1107        if dst_tex_base.aspect != dst_texture_aspects {
1108            return Err(TransferError::CopyDstMissingAspects.into());
1109        }
1110
1111        if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1112            return Err(TransferError::SampleCountNotEqual {
1113                src_sample_count: src_texture.desc.sample_count,
1114                dst_sample_count: dst_texture.desc.sample_count,
1115            }
1116            .into());
1117        }
1118
1119        // Handle texture init *before* dealing with barrier transitions so we
1120        // have an easier time inserting "immediate-inits" that may be required
1121        // by prior discards in rare cases.
1122        handle_src_texture_init(
1123            cmd_buf_data,
1124            device,
1125            source,
1126            copy_size,
1127            &src_texture,
1128            &snatch_guard,
1129        )?;
1130        handle_dst_texture_init(
1131            cmd_buf_data,
1132            device,
1133            destination,
1134            copy_size,
1135            &dst_texture,
1136            &snatch_guard,
1137        )?;
1138
1139        let src_pending = cmd_buf_data.trackers.textures.set_single(
1140            &src_texture,
1141            src_range,
1142            wgt::TextureUses::COPY_SRC,
1143        );
1144        let src_raw = src_texture.try_raw(&snatch_guard)?;
1145        src_texture
1146            .check_usage(TextureUsages::COPY_SRC)
1147            .map_err(TransferError::MissingTextureUsage)?;
1148
1149        //TODO: try to avoid this the collection. It's needed because both
1150        // `src_pending` and `dst_pending` try to hold `trackers.textures` mutably.
1151        let mut barriers: ArrayVec<_, 2> = src_pending
1152            .map(|pending| pending.into_hal(src_raw))
1153            .collect();
1154
1155        let dst_pending = cmd_buf_data.trackers.textures.set_single(
1156            &dst_texture,
1157            dst_range,
1158            wgt::TextureUses::COPY_DST,
1159        );
1160        let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1161        dst_texture
1162            .check_usage(TextureUsages::COPY_DST)
1163            .map_err(TransferError::MissingTextureUsage)?;
1164
1165        barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1166
1167        let hal_copy_size = hal::CopyExtent {
1168            width: src_copy_size.width.min(dst_copy_size.width),
1169            height: src_copy_size.height.min(dst_copy_size.height),
1170            depth: src_copy_size.depth.min(dst_copy_size.depth),
1171        };
1172        let regions = (0..array_layer_count)
1173            .map(|rel_array_layer| {
1174                let mut src_base = src_tex_base.clone();
1175                let mut dst_base = dst_tex_base.clone();
1176                src_base.array_layer += rel_array_layer;
1177                dst_base.array_layer += rel_array_layer;
1178                hal::TextureCopy {
1179                    src_base,
1180                    dst_base,
1181                    size: hal_copy_size,
1182                }
1183            })
1184            .collect::<Vec<_>>();
1185        let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1186        unsafe {
1187            cmd_buf_raw.transition_textures(&barriers);
1188            cmd_buf_raw.copy_texture_to_texture(
1189                src_raw,
1190                wgt::TextureUses::COPY_SRC,
1191                dst_raw,
1192                &regions,
1193            );
1194        }
1195
1196        cmd_buf_data_guard.mark_successful();
1197        Ok(())
1198    }
1199}