1use alloc::{sync::Arc, vec::Vec};
2
3use arrayvec::ArrayVec;
4use thiserror::Error;
5use wgt::{BufferAddress, BufferUsages, Extent3d, TextureSelector, TextureUsages};
6
7#[cfg(feature = "trace")]
8use crate::device::trace::Command as TraceCommand;
9use crate::{
10 api_log,
11 command::{clear_texture, CommandEncoderError},
12 conv,
13 device::{Device, DeviceError, MissingDownlevelFlags},
14 global::Global,
15 id::{BufferId, CommandEncoderId, TextureId},
16 init_tracker::{
17 has_copy_partial_init_tracker_coverage, MemoryInitKind, TextureInitRange,
18 TextureInitTrackerAction,
19 },
20 resource::{
21 DestroyedResourceError, InvalidResourceError, MissingBufferUsageError,
22 MissingTextureUsageError, ParentDevice, Texture, TextureErrorDimension,
23 },
24 snatch::SnatchGuard,
25};
26
27use super::{ClearError, CommandBufferMutable};
28
29pub type TexelCopyBufferInfo = wgt::TexelCopyBufferInfo<BufferId>;
30pub type TexelCopyTextureInfo = wgt::TexelCopyTextureInfo<TextureId>;
31pub type CopyExternalImageDestInfo = wgt::CopyExternalImageDestInfo<TextureId>;
32
33#[deprecated(
34 since = "24.0.0",
35 note = "This has been renamed to `TexelCopyBufferInfo`, and will be removed in 25.0.0."
36)]
37pub type ImageCopyBuffer = wgt::TexelCopyBufferInfo<BufferId>;
38
39#[deprecated(
40 since = "24.0.0",
41 note = "This has been renamed to `TexelCopyTextureInfo`, and will be removed in 25.0.0."
42)]
43pub type ImageCopyTexture = wgt::TexelCopyTextureInfo<TextureId>;
44
45#[deprecated(
46 since = "24.0.0",
47 note = "This has been renamed to `TexelCopyTextureSourceInfo`, and will be removed in 25.0.0."
48)]
49pub type ImageCopyTextureTagged = wgt::CopyExternalImageDestInfo<TextureId>;
50
51#[derive(Clone, Copy, Debug)]
52pub enum CopySide {
53 Source,
54 Destination,
55}
56
57#[derive(Clone, Debug, Error)]
59#[non_exhaustive]
60pub enum TransferError {
61 #[error("Source and destination cannot be the same buffer")]
62 SameSourceDestinationBuffer,
63 #[error(transparent)]
64 MissingBufferUsage(#[from] MissingBufferUsageError),
65 #[error(transparent)]
66 MissingTextureUsage(#[from] MissingTextureUsageError),
67 #[error("Copy of {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} buffer of size {buffer_size}")]
68 BufferOverrun {
69 start_offset: BufferAddress,
70 end_offset: BufferAddress,
71 buffer_size: BufferAddress,
72 side: CopySide,
73 },
74 #[error("Copy of {dimension:?} {start_offset}..{end_offset} would end up overrunning the bounds of the {side:?} texture of {dimension:?} size {texture_size}")]
75 TextureOverrun {
76 start_offset: u32,
77 end_offset: u32,
78 texture_size: u32,
79 dimension: TextureErrorDimension,
80 side: CopySide,
81 },
82 #[error("Unable to select texture aspect {aspect:?} from format {format:?}")]
83 InvalidTextureAspect {
84 format: wgt::TextureFormat,
85 aspect: wgt::TextureAspect,
86 },
87 #[error("Unable to select texture mip level {level} out of {total}")]
88 InvalidTextureMipLevel { level: u32, total: u32 },
89 #[error("Texture dimension must be 2D when copying from an external texture")]
90 InvalidDimensionExternal,
91 #[error("Buffer offset {0} is not aligned to block size or `COPY_BUFFER_ALIGNMENT`")]
92 UnalignedBufferOffset(BufferAddress),
93 #[error("Copy size {0} does not respect `COPY_BUFFER_ALIGNMENT`")]
94 UnalignedCopySize(BufferAddress),
95 #[error("Copy width is not a multiple of block width")]
96 UnalignedCopyWidth,
97 #[error("Copy height is not a multiple of block height")]
98 UnalignedCopyHeight,
99 #[error("Copy origin's x component is not a multiple of block width")]
100 UnalignedCopyOriginX,
101 #[error("Copy origin's y component is not a multiple of block height")]
102 UnalignedCopyOriginY,
103 #[error("Bytes per row does not respect `COPY_BYTES_PER_ROW_ALIGNMENT`")]
104 UnalignedBytesPerRow,
105 #[error("Number of bytes per row needs to be specified since more than one row is copied")]
106 UnspecifiedBytesPerRow,
107 #[error("Number of rows per image needs to be specified since more than one image is copied")]
108 UnspecifiedRowsPerImage,
109 #[error("Number of bytes per row is less than the number of bytes in a complete row")]
110 InvalidBytesPerRow,
111 #[error("Number of rows per image is invalid")]
112 InvalidRowsPerImage,
113 #[error("Copy source aspects must refer to all aspects of the source texture format")]
114 CopySrcMissingAspects,
115 #[error(
116 "Copy destination aspects must refer to all aspects of the destination texture format"
117 )]
118 CopyDstMissingAspects,
119 #[error("Copy aspect must refer to a single aspect of texture format")]
120 CopyAspectNotOne,
121 #[error("Copying from textures with format {format:?} and aspect {aspect:?} is forbidden")]
122 CopyFromForbiddenTextureFormat {
123 format: wgt::TextureFormat,
124 aspect: wgt::TextureAspect,
125 },
126 #[error("Copying to textures with format {format:?} and aspect {aspect:?} is forbidden")]
127 CopyToForbiddenTextureFormat {
128 format: wgt::TextureFormat,
129 aspect: wgt::TextureAspect,
130 },
131 #[error(
132 "Copying to textures with format {0:?} is forbidden when copying from external texture"
133 )]
134 ExternalCopyToForbiddenTextureFormat(wgt::TextureFormat),
135 #[error(
136 "Source format ({src_format:?}) and destination format ({dst_format:?}) are not copy-compatible (they may only differ in srgb-ness)"
137 )]
138 TextureFormatsNotCopyCompatible {
139 src_format: wgt::TextureFormat,
140 dst_format: wgt::TextureFormat,
141 },
142 #[error(transparent)]
143 MemoryInitFailure(#[from] ClearError),
144 #[error("Cannot encode this copy because of a missing downelevel flag")]
145 MissingDownlevelFlags(#[from] MissingDownlevelFlags),
146 #[error("Source texture sample count must be 1, got {sample_count}")]
147 InvalidSampleCount { sample_count: u32 },
148 #[error(
149 "Source sample count ({src_sample_count:?}) and destination sample count ({dst_sample_count:?}) are not equal"
150 )]
151 SampleCountNotEqual {
152 src_sample_count: u32,
153 dst_sample_count: u32,
154 },
155 #[error("Requested mip level {requested} does no exist (count: {count})")]
156 InvalidMipLevel { requested: u32, count: u32 },
157}
158
159#[derive(Clone, Debug, Error)]
161#[non_exhaustive]
162pub enum CopyError {
163 #[error(transparent)]
164 Encoder(#[from] CommandEncoderError),
165 #[error("Copy error")]
166 Transfer(#[from] TransferError),
167 #[error(transparent)]
168 DestroyedResource(#[from] DestroyedResourceError),
169 #[error(transparent)]
170 InvalidResource(#[from] InvalidResourceError),
171}
172
173impl From<DeviceError> for CopyError {
174 fn from(err: DeviceError) -> Self {
175 CopyError::Encoder(CommandEncoderError::Device(err))
176 }
177}
178
179pub(crate) fn extract_texture_selector<T>(
180 copy_texture: &wgt::TexelCopyTextureInfo<T>,
181 copy_size: &Extent3d,
182 texture: &Texture,
183) -> Result<(TextureSelector, hal::TextureCopyBase), TransferError> {
184 let format = texture.desc.format;
185 let copy_aspect = hal::FormatAspects::new(format, copy_texture.aspect);
186 if copy_aspect.is_empty() {
187 return Err(TransferError::InvalidTextureAspect {
188 format,
189 aspect: copy_texture.aspect,
190 });
191 }
192
193 let (layers, origin_z) = match texture.desc.dimension {
194 wgt::TextureDimension::D1 => (0..1, 0),
195 wgt::TextureDimension::D2 => (
196 copy_texture.origin.z..copy_texture.origin.z + copy_size.depth_or_array_layers,
197 0,
198 ),
199 wgt::TextureDimension::D3 => (0..1, copy_texture.origin.z),
200 };
201 let base = hal::TextureCopyBase {
202 origin: wgt::Origin3d {
203 x: copy_texture.origin.x,
204 y: copy_texture.origin.y,
205 z: origin_z,
206 },
207 array_layer: layers.start,
209 mip_level: copy_texture.mip_level,
210 aspect: copy_aspect,
211 };
212 let selector = TextureSelector {
213 mips: copy_texture.mip_level..copy_texture.mip_level + 1,
214 layers,
215 };
216
217 Ok((selector, base))
218}
219
220pub(crate) fn validate_linear_texture_data(
230 layout: &wgt::TexelCopyBufferLayout,
231 format: wgt::TextureFormat,
232 aspect: wgt::TextureAspect,
233 buffer_size: BufferAddress,
234 buffer_side: CopySide,
235 copy_size: &Extent3d,
236 need_copy_aligned_rows: bool,
237) -> Result<(BufferAddress, BufferAddress), TransferError> {
238 let copy_width = copy_size.width as BufferAddress;
243 let copy_height = copy_size.height as BufferAddress;
244 let depth_or_array_layers = copy_size.depth_or_array_layers as BufferAddress;
245
246 let offset = layout.offset;
247
248 let block_size = format.block_copy_size(Some(aspect)).unwrap() as BufferAddress;
249 let (block_width, block_height) = format.block_dimensions();
250 let block_width = block_width as BufferAddress;
251 let block_height = block_height as BufferAddress;
252
253 if copy_width % block_width != 0 {
254 return Err(TransferError::UnalignedCopyWidth);
255 }
256 if copy_height % block_height != 0 {
257 return Err(TransferError::UnalignedCopyHeight);
258 }
259
260 let width_in_blocks = copy_width / block_width;
261 let height_in_blocks = copy_height / block_height;
262
263 let bytes_in_last_row = width_in_blocks * block_size;
264
265 let bytes_per_row = if let Some(bytes_per_row) = layout.bytes_per_row {
266 let bytes_per_row = bytes_per_row as BufferAddress;
267 if bytes_per_row < bytes_in_last_row {
268 return Err(TransferError::InvalidBytesPerRow);
269 }
270 bytes_per_row
271 } else {
272 if depth_or_array_layers > 1 || height_in_blocks > 1 {
273 return Err(TransferError::UnspecifiedBytesPerRow);
274 }
275 0
276 };
277 let rows_per_image = if let Some(rows_per_image) = layout.rows_per_image {
278 let rows_per_image = rows_per_image as BufferAddress;
279 if rows_per_image < height_in_blocks {
280 return Err(TransferError::InvalidRowsPerImage);
281 }
282 rows_per_image
283 } else {
284 if depth_or_array_layers > 1 {
285 return Err(TransferError::UnspecifiedRowsPerImage);
286 }
287 0
288 };
289
290 if need_copy_aligned_rows {
291 let bytes_per_row_alignment = wgt::COPY_BYTES_PER_ROW_ALIGNMENT as BufferAddress;
292
293 let mut offset_alignment = block_size;
294 if format.is_depth_stencil_format() {
295 offset_alignment = 4
296 }
297 if offset % offset_alignment != 0 {
298 return Err(TransferError::UnalignedBufferOffset(offset));
299 }
300
301 if bytes_per_row % bytes_per_row_alignment != 0 {
302 return Err(TransferError::UnalignedBytesPerRow);
303 }
304 }
305
306 let bytes_per_image = bytes_per_row * rows_per_image;
307
308 let required_bytes_in_copy = if depth_or_array_layers == 0 {
309 0
310 } else {
311 let mut required_bytes_in_copy = bytes_per_image * (depth_or_array_layers - 1);
312 if height_in_blocks > 0 {
313 required_bytes_in_copy += bytes_per_row * (height_in_blocks - 1) + bytes_in_last_row;
314 }
315 required_bytes_in_copy
316 };
317
318 if offset + required_bytes_in_copy > buffer_size {
319 return Err(TransferError::BufferOverrun {
320 start_offset: offset,
321 end_offset: offset + required_bytes_in_copy,
322 buffer_size,
323 side: buffer_side,
324 });
325 }
326
327 Ok((required_bytes_in_copy, bytes_per_image))
328}
329
330pub(crate) fn validate_texture_copy_range<T>(
338 texture_copy_view: &wgt::TexelCopyTextureInfo<T>,
339 desc: &wgt::TextureDescriptor<(), Vec<wgt::TextureFormat>>,
340 texture_side: CopySide,
341 copy_size: &Extent3d,
342) -> Result<(hal::CopyExtent, u32), TransferError> {
343 let (block_width, block_height) = desc.format.block_dimensions();
344
345 let extent_virtual = desc.mip_level_size(texture_copy_view.mip_level).ok_or(
346 TransferError::InvalidTextureMipLevel {
347 level: texture_copy_view.mip_level,
348 total: desc.mip_level_count,
349 },
350 )?;
351 let extent = extent_virtual.physical_size(desc.format);
353
354 fn check_dimension(
357 dimension: TextureErrorDimension,
358 side: CopySide,
359 start_offset: u32,
360 size: u32,
361 texture_size: u32,
362 ) -> Result<(), TransferError> {
363 if start_offset <= texture_size && size <= texture_size - start_offset {
366 Ok(())
367 } else {
368 Err(TransferError::TextureOverrun {
369 start_offset,
370 end_offset: start_offset.wrapping_add(size),
371 texture_size,
372 dimension,
373 side,
374 })
375 }
376 }
377
378 check_dimension(
379 TextureErrorDimension::X,
380 texture_side,
381 texture_copy_view.origin.x,
382 copy_size.width,
383 extent.width,
384 )?;
385 check_dimension(
386 TextureErrorDimension::Y,
387 texture_side,
388 texture_copy_view.origin.y,
389 copy_size.height,
390 extent.height,
391 )?;
392 check_dimension(
393 TextureErrorDimension::Z,
394 texture_side,
395 texture_copy_view.origin.z,
396 copy_size.depth_or_array_layers,
397 extent.depth_or_array_layers,
398 )?;
399
400 if texture_copy_view.origin.x % block_width != 0 {
401 return Err(TransferError::UnalignedCopyOriginX);
402 }
403 if texture_copy_view.origin.y % block_height != 0 {
404 return Err(TransferError::UnalignedCopyOriginY);
405 }
406 if copy_size.width % block_width != 0 {
407 return Err(TransferError::UnalignedCopyWidth);
408 }
409 if copy_size.height % block_height != 0 {
410 return Err(TransferError::UnalignedCopyHeight);
411 }
412
413 let (depth, array_layer_count) = match desc.dimension {
414 wgt::TextureDimension::D1 => (1, 1),
415 wgt::TextureDimension::D2 => (1, copy_size.depth_or_array_layers),
416 wgt::TextureDimension::D3 => (copy_size.depth_or_array_layers, 1),
417 };
418
419 let copy_extent = hal::CopyExtent {
420 width: copy_size.width,
421 height: copy_size.height,
422 depth,
423 };
424 Ok((copy_extent, array_layer_count))
425}
426
427fn handle_texture_init(
428 init_kind: MemoryInitKind,
429 cmd_buf_data: &mut CommandBufferMutable,
430 device: &Device,
431 copy_texture: &TexelCopyTextureInfo,
432 copy_size: &Extent3d,
433 texture: &Arc<Texture>,
434 snatch_guard: &SnatchGuard<'_>,
435) -> Result<(), ClearError> {
436 let init_action = TextureInitTrackerAction {
437 texture: texture.clone(),
438 range: TextureInitRange {
439 mip_range: copy_texture.mip_level..copy_texture.mip_level + 1,
440 layer_range: copy_texture.origin.z
441 ..(copy_texture.origin.z + copy_size.depth_or_array_layers),
442 },
443 kind: init_kind,
444 };
445
446 let immediate_inits = cmd_buf_data
448 .texture_memory_actions
449 .register_init_action(&{ init_action });
450
451 if !immediate_inits.is_empty() {
453 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
454 for init in immediate_inits {
455 clear_texture(
456 &init.texture,
457 TextureInitRange {
458 mip_range: init.mip_level..(init.mip_level + 1),
459 layer_range: init.layer..(init.layer + 1),
460 },
461 cmd_buf_raw,
462 &mut cmd_buf_data.trackers.textures,
463 &device.alignments,
464 device.zero_buffer.as_ref(),
465 snatch_guard,
466 )?;
467 }
468 }
469
470 Ok(())
471}
472
473fn handle_src_texture_init(
478 cmd_buf_data: &mut CommandBufferMutable,
479 device: &Device,
480 source: &TexelCopyTextureInfo,
481 copy_size: &Extent3d,
482 texture: &Arc<Texture>,
483 snatch_guard: &SnatchGuard<'_>,
484) -> Result<(), TransferError> {
485 handle_texture_init(
486 MemoryInitKind::NeedsInitializedMemory,
487 cmd_buf_data,
488 device,
489 source,
490 copy_size,
491 texture,
492 snatch_guard,
493 )?;
494 Ok(())
495}
496
497fn handle_dst_texture_init(
502 cmd_buf_data: &mut CommandBufferMutable,
503 device: &Device,
504 destination: &TexelCopyTextureInfo,
505 copy_size: &Extent3d,
506 texture: &Arc<Texture>,
507 snatch_guard: &SnatchGuard<'_>,
508) -> Result<(), TransferError> {
509 let dst_init_kind = if has_copy_partial_init_tracker_coverage(
514 copy_size,
515 destination.mip_level,
516 &texture.desc,
517 ) {
518 MemoryInitKind::NeedsInitializedMemory
519 } else {
520 MemoryInitKind::ImplicitlyInitialized
521 };
522
523 handle_texture_init(
524 dst_init_kind,
525 cmd_buf_data,
526 device,
527 destination,
528 copy_size,
529 texture,
530 snatch_guard,
531 )?;
532 Ok(())
533}
534
535impl Global {
536 pub fn command_encoder_copy_buffer_to_buffer(
537 &self,
538 command_encoder_id: CommandEncoderId,
539 source: BufferId,
540 source_offset: BufferAddress,
541 destination: BufferId,
542 destination_offset: BufferAddress,
543 size: BufferAddress,
544 ) -> Result<(), CopyError> {
545 profiling::scope!("CommandEncoder::copy_buffer_to_buffer");
546 api_log!(
547 "CommandEncoder::copy_buffer_to_buffer {source:?} -> {destination:?} {size:?}bytes"
548 );
549
550 if source == destination {
551 return Err(TransferError::SameSourceDestinationBuffer.into());
552 }
553 let hub = &self.hub;
554
555 let cmd_buf = hub
556 .command_buffers
557 .get(command_encoder_id.into_command_buffer_id());
558 let mut cmd_buf_data = cmd_buf.data.lock();
559 let mut cmd_buf_data_guard = cmd_buf_data.record()?;
560 let cmd_buf_data = &mut *cmd_buf_data_guard;
561
562 let device = &cmd_buf.device;
563 device.check_is_valid()?;
564
565 #[cfg(feature = "trace")]
566 if let Some(ref mut list) = cmd_buf_data.commands {
567 list.push(TraceCommand::CopyBufferToBuffer {
568 src: source,
569 src_offset: source_offset,
570 dst: destination,
571 dst_offset: destination_offset,
572 size,
573 });
574 }
575
576 let snatch_guard = device.snatchable_lock.read();
577
578 let src_buffer = hub.buffers.get(source).get()?;
579
580 src_buffer.same_device_as(cmd_buf.as_ref())?;
581
582 let src_pending = cmd_buf_data
583 .trackers
584 .buffers
585 .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
586
587 let src_raw = src_buffer.try_raw(&snatch_guard)?;
588 src_buffer
589 .check_usage(BufferUsages::COPY_SRC)
590 .map_err(TransferError::MissingBufferUsage)?;
591 let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
593
594 let dst_buffer = hub.buffers.get(destination).get()?;
595
596 dst_buffer.same_device_as(cmd_buf.as_ref())?;
597
598 let dst_pending = cmd_buf_data
599 .trackers
600 .buffers
601 .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
602
603 let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
604 dst_buffer
605 .check_usage(BufferUsages::COPY_DST)
606 .map_err(TransferError::MissingBufferUsage)?;
607 let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
608
609 if size % wgt::COPY_BUFFER_ALIGNMENT != 0 {
610 return Err(TransferError::UnalignedCopySize(size).into());
611 }
612 if source_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
613 return Err(TransferError::UnalignedBufferOffset(source_offset).into());
614 }
615 if destination_offset % wgt::COPY_BUFFER_ALIGNMENT != 0 {
616 return Err(TransferError::UnalignedBufferOffset(destination_offset).into());
617 }
618 if !device
619 .downlevel
620 .flags
621 .contains(wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER)
622 && (src_buffer.usage.contains(BufferUsages::INDEX)
623 || dst_buffer.usage.contains(BufferUsages::INDEX))
624 {
625 let forbidden_usages = BufferUsages::VERTEX
626 | BufferUsages::UNIFORM
627 | BufferUsages::INDIRECT
628 | BufferUsages::STORAGE;
629 if src_buffer.usage.intersects(forbidden_usages)
630 || dst_buffer.usage.intersects(forbidden_usages)
631 {
632 return Err(TransferError::MissingDownlevelFlags(MissingDownlevelFlags(
633 wgt::DownlevelFlags::UNRESTRICTED_INDEX_BUFFER,
634 ))
635 .into());
636 }
637 }
638
639 let source_end_offset = source_offset + size;
640 let destination_end_offset = destination_offset + size;
641 if source_end_offset > src_buffer.size {
642 return Err(TransferError::BufferOverrun {
643 start_offset: source_offset,
644 end_offset: source_end_offset,
645 buffer_size: src_buffer.size,
646 side: CopySide::Source,
647 }
648 .into());
649 }
650 if destination_end_offset > dst_buffer.size {
651 return Err(TransferError::BufferOverrun {
652 start_offset: destination_offset,
653 end_offset: destination_end_offset,
654 buffer_size: dst_buffer.size,
655 side: CopySide::Destination,
656 }
657 .into());
658 }
659
660 if size == 0 {
661 log::trace!("Ignoring copy_buffer_to_buffer of size 0");
662 cmd_buf_data_guard.mark_successful();
663 return Ok(());
664 }
665
666 cmd_buf_data.buffer_memory_init_actions.extend(
668 dst_buffer.initialization_status.read().create_action(
669 &dst_buffer,
670 destination_offset..(destination_offset + size),
671 MemoryInitKind::ImplicitlyInitialized,
672 ),
673 );
674 cmd_buf_data.buffer_memory_init_actions.extend(
675 src_buffer.initialization_status.read().create_action(
676 &src_buffer,
677 source_offset..(source_offset + size),
678 MemoryInitKind::NeedsInitializedMemory,
679 ),
680 );
681
682 let region = hal::BufferCopy {
683 src_offset: source_offset,
684 dst_offset: destination_offset,
685 size: wgt::BufferSize::new(size).unwrap(),
686 };
687 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
688 let barriers = src_barrier
689 .into_iter()
690 .chain(dst_barrier)
691 .collect::<Vec<_>>();
692 unsafe {
693 cmd_buf_raw.transition_buffers(&barriers);
694 cmd_buf_raw.copy_buffer_to_buffer(src_raw, dst_raw, &[region]);
695 }
696
697 cmd_buf_data_guard.mark_successful();
698 Ok(())
699 }
700
701 pub fn command_encoder_copy_buffer_to_texture(
702 &self,
703 command_encoder_id: CommandEncoderId,
704 source: &TexelCopyBufferInfo,
705 destination: &TexelCopyTextureInfo,
706 copy_size: &Extent3d,
707 ) -> Result<(), CopyError> {
708 profiling::scope!("CommandEncoder::copy_buffer_to_texture");
709 api_log!(
710 "CommandEncoder::copy_buffer_to_texture {:?} -> {:?} {copy_size:?}",
711 source.buffer,
712 destination.texture
713 );
714
715 let hub = &self.hub;
716
717 let cmd_buf = hub
718 .command_buffers
719 .get(command_encoder_id.into_command_buffer_id());
720 let mut cmd_buf_data = cmd_buf.data.lock();
721 let mut cmd_buf_data_guard = cmd_buf_data.record()?;
722 let cmd_buf_data = &mut *cmd_buf_data_guard;
723
724 let device = &cmd_buf.device;
725 device.check_is_valid()?;
726
727 #[cfg(feature = "trace")]
728 if let Some(ref mut list) = cmd_buf_data.commands {
729 list.push(TraceCommand::CopyBufferToTexture {
730 src: *source,
731 dst: *destination,
732 size: *copy_size,
733 });
734 }
735
736 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
737 log::trace!("Ignoring copy_buffer_to_texture of size 0");
738 cmd_buf_data_guard.mark_successful();
739 return Ok(());
740 }
741
742 let dst_texture = hub.textures.get(destination.texture).get()?;
743
744 dst_texture.same_device_as(cmd_buf.as_ref())?;
745
746 let (hal_copy_size, array_layer_count) = validate_texture_copy_range(
747 destination,
748 &dst_texture.desc,
749 CopySide::Destination,
750 copy_size,
751 )?;
752
753 let (dst_range, dst_base) = extract_texture_selector(destination, copy_size, &dst_texture)?;
754
755 let snatch_guard = device.snatchable_lock.read();
756
757 handle_dst_texture_init(
761 cmd_buf_data,
762 device,
763 destination,
764 copy_size,
765 &dst_texture,
766 &snatch_guard,
767 )?;
768
769 let src_buffer = hub.buffers.get(source.buffer).get()?;
770
771 src_buffer.same_device_as(cmd_buf.as_ref())?;
772
773 let src_pending = cmd_buf_data
774 .trackers
775 .buffers
776 .set_single(&src_buffer, wgt::BufferUses::COPY_SRC);
777
778 let src_raw = src_buffer.try_raw(&snatch_guard)?;
779 src_buffer
780 .check_usage(BufferUsages::COPY_SRC)
781 .map_err(TransferError::MissingBufferUsage)?;
782 let src_barrier = src_pending.map(|pending| pending.into_hal(&src_buffer, &snatch_guard));
783
784 let dst_pending = cmd_buf_data.trackers.textures.set_single(
785 &dst_texture,
786 dst_range,
787 wgt::TextureUses::COPY_DST,
788 );
789 let dst_raw = dst_texture.try_raw(&snatch_guard)?;
790 dst_texture
791 .check_usage(TextureUsages::COPY_DST)
792 .map_err(TransferError::MissingTextureUsage)?;
793 let dst_barrier = dst_pending
794 .map(|pending| pending.into_hal(dst_raw))
795 .collect::<Vec<_>>();
796
797 if !dst_base.aspect.is_one() {
798 return Err(TransferError::CopyAspectNotOne.into());
799 }
800
801 if !conv::is_valid_copy_dst_texture_format(dst_texture.desc.format, destination.aspect) {
802 return Err(TransferError::CopyToForbiddenTextureFormat {
803 format: dst_texture.desc.format,
804 aspect: destination.aspect,
805 }
806 .into());
807 }
808
809 let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
810 &source.layout,
811 dst_texture.desc.format,
812 destination.aspect,
813 src_buffer.size,
814 CopySide::Source,
815 copy_size,
816 true,
817 )?;
818
819 if dst_texture.desc.format.is_depth_stencil_format() {
820 device
821 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
822 .map_err(TransferError::from)?;
823 }
824
825 cmd_buf_data.buffer_memory_init_actions.extend(
826 src_buffer.initialization_status.read().create_action(
827 &src_buffer,
828 source.layout.offset..(source.layout.offset + required_buffer_bytes_in_copy),
829 MemoryInitKind::NeedsInitializedMemory,
830 ),
831 );
832
833 let regions = (0..array_layer_count)
834 .map(|rel_array_layer| {
835 let mut texture_base = dst_base.clone();
836 texture_base.array_layer += rel_array_layer;
837 let mut buffer_layout = source.layout;
838 buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
839 hal::BufferTextureCopy {
840 buffer_layout,
841 texture_base,
842 size: hal_copy_size,
843 }
844 })
845 .collect::<Vec<_>>();
846
847 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
848 unsafe {
849 cmd_buf_raw.transition_textures(&dst_barrier);
850 cmd_buf_raw.transition_buffers(src_barrier.as_slice());
851 cmd_buf_raw.copy_buffer_to_texture(src_raw, dst_raw, ®ions);
852 }
853
854 cmd_buf_data_guard.mark_successful();
855 Ok(())
856 }
857
858 pub fn command_encoder_copy_texture_to_buffer(
859 &self,
860 command_encoder_id: CommandEncoderId,
861 source: &TexelCopyTextureInfo,
862 destination: &TexelCopyBufferInfo,
863 copy_size: &Extent3d,
864 ) -> Result<(), CopyError> {
865 profiling::scope!("CommandEncoder::copy_texture_to_buffer");
866 api_log!(
867 "CommandEncoder::copy_texture_to_buffer {:?} -> {:?} {copy_size:?}",
868 source.texture,
869 destination.buffer
870 );
871
872 let hub = &self.hub;
873
874 let cmd_buf = hub
875 .command_buffers
876 .get(command_encoder_id.into_command_buffer_id());
877 let mut cmd_buf_data = cmd_buf.data.lock();
878 let mut cmd_buf_data_guard = cmd_buf_data.record()?;
879 let cmd_buf_data = &mut *cmd_buf_data_guard;
880
881 let device = &cmd_buf.device;
882 device.check_is_valid()?;
883
884 #[cfg(feature = "trace")]
885 if let Some(list) = cmd_buf_data.commands.as_mut() {
886 list.push(TraceCommand::CopyTextureToBuffer {
887 src: *source,
888 dst: *destination,
889 size: *copy_size,
890 });
891 }
892
893 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
894 log::trace!("Ignoring copy_texture_to_buffer of size 0");
895 cmd_buf_data_guard.mark_successful();
896 return Ok(());
897 }
898
899 let src_texture = hub.textures.get(source.texture).get()?;
900
901 src_texture.same_device_as(cmd_buf.as_ref())?;
902
903 let (hal_copy_size, array_layer_count) =
904 validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
905
906 let (src_range, src_base) = extract_texture_selector(source, copy_size, &src_texture)?;
907
908 let snatch_guard = device.snatchable_lock.read();
909
910 handle_src_texture_init(
914 cmd_buf_data,
915 device,
916 source,
917 copy_size,
918 &src_texture,
919 &snatch_guard,
920 )?;
921
922 let src_pending = cmd_buf_data.trackers.textures.set_single(
923 &src_texture,
924 src_range,
925 wgt::TextureUses::COPY_SRC,
926 );
927 let src_raw = src_texture.try_raw(&snatch_guard)?;
928 src_texture
929 .check_usage(TextureUsages::COPY_SRC)
930 .map_err(TransferError::MissingTextureUsage)?;
931 if src_texture.desc.sample_count != 1 {
932 return Err(TransferError::InvalidSampleCount {
933 sample_count: src_texture.desc.sample_count,
934 }
935 .into());
936 }
937 if source.mip_level >= src_texture.desc.mip_level_count {
938 return Err(TransferError::InvalidMipLevel {
939 requested: source.mip_level,
940 count: src_texture.desc.mip_level_count,
941 }
942 .into());
943 }
944 let src_barrier = src_pending
945 .map(|pending| pending.into_hal(src_raw))
946 .collect::<Vec<_>>();
947
948 let dst_buffer = hub.buffers.get(destination.buffer).get()?;
949
950 dst_buffer.same_device_as(cmd_buf.as_ref())?;
951
952 let dst_pending = cmd_buf_data
953 .trackers
954 .buffers
955 .set_single(&dst_buffer, wgt::BufferUses::COPY_DST);
956
957 let dst_raw = dst_buffer.try_raw(&snatch_guard)?;
958 dst_buffer
959 .check_usage(BufferUsages::COPY_DST)
960 .map_err(TransferError::MissingBufferUsage)?;
961 let dst_barrier = dst_pending.map(|pending| pending.into_hal(&dst_buffer, &snatch_guard));
962
963 if !src_base.aspect.is_one() {
964 return Err(TransferError::CopyAspectNotOne.into());
965 }
966
967 if !conv::is_valid_copy_src_texture_format(src_texture.desc.format, source.aspect) {
968 return Err(TransferError::CopyFromForbiddenTextureFormat {
969 format: src_texture.desc.format,
970 aspect: source.aspect,
971 }
972 .into());
973 }
974
975 let (required_buffer_bytes_in_copy, bytes_per_array_layer) = validate_linear_texture_data(
976 &destination.layout,
977 src_texture.desc.format,
978 source.aspect,
979 dst_buffer.size,
980 CopySide::Destination,
981 copy_size,
982 true,
983 )?;
984
985 if src_texture.desc.format.is_depth_stencil_format() {
986 device
987 .require_downlevel_flags(wgt::DownlevelFlags::DEPTH_TEXTURE_AND_BUFFER_COPIES)
988 .map_err(TransferError::from)?;
989 }
990
991 cmd_buf_data.buffer_memory_init_actions.extend(
992 dst_buffer.initialization_status.read().create_action(
993 &dst_buffer,
994 destination.layout.offset
995 ..(destination.layout.offset + required_buffer_bytes_in_copy),
996 MemoryInitKind::ImplicitlyInitialized,
997 ),
998 );
999
1000 let regions = (0..array_layer_count)
1001 .map(|rel_array_layer| {
1002 let mut texture_base = src_base.clone();
1003 texture_base.array_layer += rel_array_layer;
1004 let mut buffer_layout = destination.layout;
1005 buffer_layout.offset += rel_array_layer as u64 * bytes_per_array_layer;
1006 hal::BufferTextureCopy {
1007 buffer_layout,
1008 texture_base,
1009 size: hal_copy_size,
1010 }
1011 })
1012 .collect::<Vec<_>>();
1013 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1014 unsafe {
1015 cmd_buf_raw.transition_buffers(dst_barrier.as_slice());
1016 cmd_buf_raw.transition_textures(&src_barrier);
1017 cmd_buf_raw.copy_texture_to_buffer(
1018 src_raw,
1019 wgt::TextureUses::COPY_SRC,
1020 dst_raw,
1021 ®ions,
1022 );
1023 }
1024
1025 cmd_buf_data_guard.mark_successful();
1026 Ok(())
1027 }
1028
1029 pub fn command_encoder_copy_texture_to_texture(
1030 &self,
1031 command_encoder_id: CommandEncoderId,
1032 source: &TexelCopyTextureInfo,
1033 destination: &TexelCopyTextureInfo,
1034 copy_size: &Extent3d,
1035 ) -> Result<(), CopyError> {
1036 profiling::scope!("CommandEncoder::copy_texture_to_texture");
1037 api_log!(
1038 "CommandEncoder::copy_texture_to_texture {:?} -> {:?} {copy_size:?}",
1039 source.texture,
1040 destination.texture
1041 );
1042
1043 let hub = &self.hub;
1044
1045 let cmd_buf = hub
1046 .command_buffers
1047 .get(command_encoder_id.into_command_buffer_id());
1048 let mut cmd_buf_data = cmd_buf.data.lock();
1049 let mut cmd_buf_data_guard = cmd_buf_data.record()?;
1050 let cmd_buf_data = &mut *cmd_buf_data_guard;
1051
1052 let device = &cmd_buf.device;
1053 device.check_is_valid()?;
1054
1055 let snatch_guard = device.snatchable_lock.read();
1056
1057 #[cfg(feature = "trace")]
1058 if let Some(ref mut list) = cmd_buf_data.commands {
1059 list.push(TraceCommand::CopyTextureToTexture {
1060 src: *source,
1061 dst: *destination,
1062 size: *copy_size,
1063 });
1064 }
1065
1066 if copy_size.width == 0 || copy_size.height == 0 || copy_size.depth_or_array_layers == 0 {
1067 log::trace!("Ignoring copy_texture_to_texture of size 0");
1068 cmd_buf_data_guard.mark_successful();
1069 return Ok(());
1070 }
1071
1072 let src_texture = hub.textures.get(source.texture).get()?;
1073 let dst_texture = hub.textures.get(destination.texture).get()?;
1074
1075 src_texture.same_device_as(cmd_buf.as_ref())?;
1076 dst_texture.same_device_as(cmd_buf.as_ref())?;
1077
1078 if src_texture.desc.format.remove_srgb_suffix()
1081 != dst_texture.desc.format.remove_srgb_suffix()
1082 {
1083 return Err(TransferError::TextureFormatsNotCopyCompatible {
1084 src_format: src_texture.desc.format,
1085 dst_format: dst_texture.desc.format,
1086 }
1087 .into());
1088 }
1089
1090 let (src_copy_size, array_layer_count) =
1091 validate_texture_copy_range(source, &src_texture.desc, CopySide::Source, copy_size)?;
1092 let (dst_copy_size, _) = validate_texture_copy_range(
1093 destination,
1094 &dst_texture.desc,
1095 CopySide::Destination,
1096 copy_size,
1097 )?;
1098
1099 let (src_range, src_tex_base) = extract_texture_selector(source, copy_size, &src_texture)?;
1100 let (dst_range, dst_tex_base) =
1101 extract_texture_selector(destination, copy_size, &dst_texture)?;
1102 let src_texture_aspects = hal::FormatAspects::from(src_texture.desc.format);
1103 let dst_texture_aspects = hal::FormatAspects::from(dst_texture.desc.format);
1104 if src_tex_base.aspect != src_texture_aspects {
1105 return Err(TransferError::CopySrcMissingAspects.into());
1106 }
1107 if dst_tex_base.aspect != dst_texture_aspects {
1108 return Err(TransferError::CopyDstMissingAspects.into());
1109 }
1110
1111 if src_texture.desc.sample_count != dst_texture.desc.sample_count {
1112 return Err(TransferError::SampleCountNotEqual {
1113 src_sample_count: src_texture.desc.sample_count,
1114 dst_sample_count: dst_texture.desc.sample_count,
1115 }
1116 .into());
1117 }
1118
1119 handle_src_texture_init(
1123 cmd_buf_data,
1124 device,
1125 source,
1126 copy_size,
1127 &src_texture,
1128 &snatch_guard,
1129 )?;
1130 handle_dst_texture_init(
1131 cmd_buf_data,
1132 device,
1133 destination,
1134 copy_size,
1135 &dst_texture,
1136 &snatch_guard,
1137 )?;
1138
1139 let src_pending = cmd_buf_data.trackers.textures.set_single(
1140 &src_texture,
1141 src_range,
1142 wgt::TextureUses::COPY_SRC,
1143 );
1144 let src_raw = src_texture.try_raw(&snatch_guard)?;
1145 src_texture
1146 .check_usage(TextureUsages::COPY_SRC)
1147 .map_err(TransferError::MissingTextureUsage)?;
1148
1149 let mut barriers: ArrayVec<_, 2> = src_pending
1152 .map(|pending| pending.into_hal(src_raw))
1153 .collect();
1154
1155 let dst_pending = cmd_buf_data.trackers.textures.set_single(
1156 &dst_texture,
1157 dst_range,
1158 wgt::TextureUses::COPY_DST,
1159 );
1160 let dst_raw = dst_texture.try_raw(&snatch_guard)?;
1161 dst_texture
1162 .check_usage(TextureUsages::COPY_DST)
1163 .map_err(TransferError::MissingTextureUsage)?;
1164
1165 barriers.extend(dst_pending.map(|pending| pending.into_hal(dst_raw)));
1166
1167 let hal_copy_size = hal::CopyExtent {
1168 width: src_copy_size.width.min(dst_copy_size.width),
1169 height: src_copy_size.height.min(dst_copy_size.height),
1170 depth: src_copy_size.depth.min(dst_copy_size.depth),
1171 };
1172 let regions = (0..array_layer_count)
1173 .map(|rel_array_layer| {
1174 let mut src_base = src_tex_base.clone();
1175 let mut dst_base = dst_tex_base.clone();
1176 src_base.array_layer += rel_array_layer;
1177 dst_base.array_layer += rel_array_layer;
1178 hal::TextureCopy {
1179 src_base,
1180 dst_base,
1181 size: hal_copy_size,
1182 }
1183 })
1184 .collect::<Vec<_>>();
1185 let cmd_buf_raw = cmd_buf_data.encoder.open()?;
1186 unsafe {
1187 cmd_buf_raw.transition_textures(&barriers);
1188 cmd_buf_raw.copy_texture_to_texture(
1189 src_raw,
1190 wgt::TextureUses::COPY_SRC,
1191 dst_raw,
1192 ®ions,
1193 );
1194 }
1195
1196 cmd_buf_data_guard.mark_successful();
1197 Ok(())
1198 }
1199}