Unable to draw textures on SCNGeometry which is created from ARKit FaceAnchor points.

In the below code I have extracted face mesh vertices from ARKit face anchors and created a custom face mesh using SceneKit SCNGeometry. This enabled me to stretch face mesh vertices as per my requirement.

Now the problem I am facing is as follows. I am trying to apply a lipstick texture material which is of type SCNMaterial. Although ARSCNFaceGeometry lets me apply different textures through SCNMaterial and SCNNode, I am not able to do the same using mu CustomFaceGeometry. When I am applying a lipstick texture which looks like the image attached below, the full face is getting colored or modified, I want only that part of the face which has texture transparency as >0 and I dont want other part of the face to be modified.

Can you give me a detailed solution using code?

//  ViewController.swift

import UIKit
import ARKit
import SceneKit
import simd

class ViewController: UIViewController, ARSCNViewDelegate, ARSessionDelegate{
    @IBOutlet weak var sceneView: ARSCNView!
    let vertexIndicesOfInterest = [250]
    var customFaceGeometry: CustomFaceGeometry!
    var scnFaceGeometry: SCNGeometry!
    private var faceUvGenerator: FaceTextureGenerator!
    var faceGeometry: ARSCNFaceGeometry!
    override func viewDidLoad() {
        super.viewDidLoad()
        sceneView.delegate = self

    override func viewWillAppear(_ animated: Bool) {
        super.viewWillAppear(animated)
        let configuration = ARFaceTrackingConfiguration()
        sceneView.session.run(configuration)
    }
}

extension ViewController {
    func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
        guard let faceAnchor = anchor as? ARFaceAnchor else { return }
        customFaceGeometry = CustomFaceGeometry(fromFaceAnchor: faceAnchor)
        let customGeometryNode = SCNNode(geometry: customFaceGeometry.geometry)
        customFaceGeometry.geometry.firstMaterial?.fillMode = .lines
        customFaceGeometry.geometry.firstMaterial?.transparency = 0.0
        customFaceGeometry.geometry.firstMaterial?.isDoubleSided = true
        node.addChildNode(customGeometryNode)
    }

    func renderer(_ renderer: SCNSceneRenderer, willUpdate node: SCNNode, for anchor: ARAnchor) {
        guard let faceAnchor = anchor as? ARFaceAnchor,
              let faceMeshNode = node.childNodes.first else { return }
        DispatchQueue.main.async {
            self.customFaceGeometry.update(withFaceAnchor: faceAnchor, node: faceMeshNode)
            }
    }
}

class CustomFaceGeometry {
    var geometry: SCNGeometry
    let lipImage = UIImage(named: "Face.scnassets/lip_arks_y7.png")
    init(fromFaceAnchor faceAnchor: ARFaceAnchor) {
        self.geometry = CustomFaceGeometry.createCustomSCNGeometry(from: faceAnchor)!
    }
    static func createCustomFaceGeometry(fromVertices vertices_o: [SCNVector3]) -> SCNGeometry {
        var vertices = vertices_o
        let vertexData = Data(bytes: vertices, count: vertices.count * MemoryLayout<SCNVector3>.size)
        let vertexSource = SCNGeometrySource(data: vertexData,
                                             semantic: .vertex,
                                             vectorCount: vertices.count,
                                             usesFloatComponents: true,
                                             componentsPerVector: 3,
                                             bytesPerComponent: MemoryLayout<Float>.size,
                                             dataOffset: 0,
                                             dataStride: MemoryLayout<SCNVector3>.stride)  
        let indices: [Int32] = Array(0..<Int32(vertices.count))
        let indexData = Data(bytes: indices, count: indices.count * MemoryLayout<Int32>.size)
        let element = SCNGeometryElement(data: indexData, primitiveType: .point, primitiveCount: vertices.count, bytesPerIndex: MemoryLayout<Int32>.size)

        return SCNGeometry(sources: [vertexSource], elements: [element])
    }    
    static func createGeometry(fromFaceAnchor faceAnchor: ARFaceAnchor) -> SCNGeometry 
        let vertices = faceAnchor.geometry.vertices.map { SCNVector3($0.x, $0.y, $0.z) }
        return CustomFaceGeometry.createCustomFaceGeometry(fromVertices: vertices)
    }

    func update(withFaceAnchor faceAnchor: ARFaceAnchor, node: SCNNode) {
        if let newGeometry = CustomFaceGeometry.createCustomSCNGeometry(from: faceAnchor) {

            node.geometry = newGeometry
            let lipstickNode = SCNNode(geometry: newGeometry)
            let lipstickTextureMaterial = SCNMaterial()
            lipstickTextureMaterial.diffuse.contents = lipImage
            lipstickTextureMaterial.transparency = 1.0
            lipstickNode.geometry?.firstMaterial = lipstickTextureMaterial
            node.geometry?.firstMaterial?.fillMode = .lines

            node.geometry?.firstMaterial?.transparency = 0.5
            
        }
    }
    
    static func createCustomSCNGeometry(from faceAnchor: ARFaceAnchor) -> SCNGeometry? {
        let faceGeometry = faceAnchor.geometry
        var vertices: [SCNVector3] = faceGeometry.vertices.map { SCNVector3($0.x, $0.y, $0.z) }
        
        print(vertices[250])
        let ll_ratio_y = Float(0.969999)
        vertices[290] = SCNVector3(x: vertices[290].x, y: vertices[290].y*ll_ratio_y, z: vertices[290].z)
        vertices[274] = SCNVector3(x: vertices[274].x, y: vertices[274].y*ll_ratio_y, z: vertices[274].z)
        vertices[265] = SCNVector3(x: vertices[265].x, y: vertices[265].y*ll_ratio_y, z: vertices[265].z)
        vertices[700] = SCNVector3(x: vertices[700].x, y: vertices[700].y*ll_ratio_y, z: vertices[700].z)
        vertices[730] = SCNVector3(x: vertices[730].x, y: vertices[730].y*ll_ratio_y, z: vertices[730].z)
        vertices[25] = SCNVector3(x: vertices[25].x, y: vertices[25].y*ll_ratio_y, z: vertices[25].z)
        vertices[709] = SCNVector3(x: vertices[709].x, y: vertices[709].y*ll_ratio_y, z: vertices[709].z)
        vertices[725] = SCNVector3(x: vertices[725].x, y: vertices[725].y*ll_ratio_y, z: vertices[725].z)
        vertices[710] = SCNVector3(x: vertices[710].x, y: vertices[710].y*ll_ratio_y, z: vertices[710].z)
        
        
        let vertexData = Data(bytes: vertices, count: vertices.count * MemoryLayout<SCNVector3>.size)
        let vertexSource = SCNGeometrySource(data: vertexData, semantic: .vertex, vectorCount: vertices.count, usesFloatComponents: true, componentsPerVector: 3, bytesPerComponent: MemoryLayout<Float>.size, dataOffset: 0, dataStride: MemoryLayout<SCNVector3>.stride)
        let indices: [UInt16] = faceGeometry.triangleIndices.map(UInt16.init)
        let indexData = Data(bytes: indices, count: indices.count * MemoryLayout<UInt16>.size)
        let element = SCNGeometryElement(data: indexData, primitiveType: .triangles, primitiveCount: indices.count / 3, bytesPerIndex: MemoryLayout<UInt16>.size)
        return SCNGeometry(sources: [vertexSource], elements: [element])
    }
}

You need a texture source as well. Mapping the UV to this custom mesh. SCNGeometry(sources: [vertexSource, normalSource, textureSource], elements: elementArray). Makes one appreciate Blender.

@Bill3D I am trying to draw lipstick texture on ARKitFaceMesh. I have already received very good results when I am just using ARKit. But to meet a certain criteria, I need the option to modify the ARKit Face Mesh vertices. For that reason I took the vertices from ARFaceAnchor and derived normals and textureCoordinates correctly. Now using SCNode and SCNmaterial, I am trying to render the texture on my face. While colors are getting applied but I am not able to get the exact lipstick texture on my face. It is taking any random color from the texture and plotting it. I have already checked whether my UV coordinates are right or not. Turns out the UV coordinates are perfectly aligned with the ARKit Face mesh UV coordinates.

Now I am really lost regarding how can I apply texture on my custom face mesh. Any idea on this?

Unable to draw textures on SCNGeometry which is created from ARKit FaceAnchor points.
 
 
Q