Adding custom material to sceneReconstruction mesh

Question

Created Feb ’24

Replies 3

Boosts 0

Participants 4

I wanted to add a custom material over the mesh detected by the sceneReconstruction provider but i can't find a way to convert the meshAnchor to a usable MeshResource

func processReconstructionUpdates() async {
        for await update in sceneReconstruction.anchorUpdates {
            let meshAnchor = update.anchor
            
            guard let shape = try? await ShapeResource.generateStaticMesh(from: meshAnchor)             else { continue }
            
            switch update.event {
            case .added:
                let entity = ModelEntity(
                                         mesh: **somehow get the mesh from mesh anchor here**, 

                                         materials: [material]
               )       
                
                contentEntity.addChild(entity)
            case .updated:
                ...
            case .removed:
                ...
            @unknown default:
                fatalError("Unsupported anchor event")
            }
        }
    }

Answered by pj4533 in 780967022

Yeaaaah! I just figured this out today and posted on an older thread about it. The key for me was finding this repo: https://github.com/XRealityZone/what-vision-os-can-do/blob/main/WhatVisionOSCanDo/ShowCase/WorldScening/WorldSceningTrackingModel.swift#L70

Important code:

@MainActor fileprivate func generateModelEntity(geometry: MeshAnchor.Geometry) async throws -> ModelEntity {
        // generate mesh
        var desc = MeshDescriptor()
        let posValues = geometry.vertices.asSIMD3(ofType: Float.self)
        desc.positions = .init(posValues)
        let normalValues = geometry.normals.asSIMD3(ofType: Float.self)
        desc.normals = .init(normalValues)
        do {
            desc.primitives = .polygons(
                // 应该都是三角形，所以这里直接写 3
                (0..<geometry.faces.count).map { _ in UInt8(3) },
                (0..<geometry.faces.count * 3).map {
                    geometry.faces.buffer.contents()
                        .advanced(by: $0 * geometry.faces.bytesPerIndex)
                        .assumingMemoryBound(to: UInt32.self).pointee
                }
            )
        }
        let meshResource = try await MeshResource.generate(from: [desc])
        let material = SimpleMaterial(color: .red, isMetallic: false)
        let modelEntity = ModelEntity(mesh: meshResource, materials: [material])
        return modelEntity
    }

Boost

Answer 1

pj4533 OP

Feb ’24

Accepted Answer

Yeaaaah! I just figured this out today and posted on an older thread about it. The key for me was finding this repo: https://github.com/XRealityZone/what-vision-os-can-do/blob/main/WhatVisionOSCanDo/ShowCase/WorldScening/WorldSceningTrackingModel.swift#L70

Important code:

@MainActor fileprivate func generateModelEntity(geometry: MeshAnchor.Geometry) async throws -> ModelEntity {
        // generate mesh
        var desc = MeshDescriptor()
        let posValues = geometry.vertices.asSIMD3(ofType: Float.self)
        desc.positions = .init(posValues)
        let normalValues = geometry.normals.asSIMD3(ofType: Float.self)
        desc.normals = .init(normalValues)
        do {
            desc.primitives = .polygons(
                // 应该都是三角形，所以这里直接写 3
                (0..<geometry.faces.count).map { _ in UInt8(3) },
                (0..<geometry.faces.count * 3).map {
                    geometry.faces.buffer.contents()
                        .advanced(by: $0 * geometry.faces.bytesPerIndex)
                        .assumingMemoryBound(to: UInt32.self).pointee
                }
            )
        }
        let meshResource = try await MeshResource.generate(from: [desc])
        let material = SimpleMaterial(color: .red, isMetallic: false)
        let modelEntity = ModelEntity(mesh: meshResource, materials: [material])
        return modelEntity
    }

1

Answer 2

stevecfox OP

Jun ’24

I am also trying to do this, and this code looks good. But I am getting GeometrySource has no member 'asSIMD3' for the let posValues = geometry.vertices.asSIMD3(ofType: Float.self) line in XCode 15 for Vision OS 1.0. Wondering if this was for an earlier beta of Vision OS or something?

0

Answer 3

ViggoOveres OP

Aug ’24

@stevecfox Leaving this here in case someone else also wonders this, but coming from the GitHub the 'asSIMD3' is an extension of GeometrySource:

extension GeometrySource {
    @MainActor func asArray<T>(ofType: T.Type) -> [T] {
        assert(MemoryLayout<T>.stride == stride, "Invalid stride \(MemoryLayout<T>.stride); expected \(stride)")
        return (0..<self.count).map {
            buffer.contents().advanced(by: offset + stride * Int($0)).assumingMemoryBound(to: T.self).pointee
        }
    }

    // SIMD3 has the same storage as SIMD4.
    @MainActor  func asSIMD3<T>(ofType: T.Type) -> [SIMD3<T>] {
        return asArray(ofType: (T, T, T).self).map { .init($0.0, $0.1, $0.2) }
    }
}

0