List Cell - Modifying associated object causes reload of whole list

I've got a List containing Colour objects. Each colour may have an associated Project Colour object.

What I'm trying to do is set it up so that you can tap a cell and it will add/remove a project colour.

The adding/removing is working, but each time I do so, it appears the whole view is reloaded, the scroll position is reset and any predicate is removed.

This code I have so far

List {
            ForEach(colourList) { section in
                let header : String = section.id
                Section(header: Text(header)) {
                    ForEach(section) { colour in
                        HStack {
                            if checkIfProjectColour(colour: colour) {
                                Image(systemName: "checkmark")
                            }
                            VStack(alignment: .leading){
                                HStack {
                                    if let name = colour.name {
                                        Text(name)
                                    }
                                }
                            }
                            Spacer()
                        }
                        .contentShape(Rectangle())
                        .onTapGesture {
                            if checkIfProjectColour(colour: colour) {
                                removeProjectColour(colour: colour)
                            } else {
                                addProjectColour(colour: colour)
                            }
                        }
                    }
                }
            }
            .onAppear() {
                filters = appSetting.filters
                colourList.nsPredicate = getFilterPredicate()
                print("predicate: on appear - \(String(describing: getFilterPredicate()))")
            }
            .refreshable {
                viewContext.refreshAllObjects()
            }
        }
        .searchable(text: $searchText)
        .onSubmit(of: .search) {
            colourList.nsPredicate = getFilterPredicate()
        }
        .onChange(of: searchText) {
            colourList.nsPredicate = getFilterPredicate()
        }

The checkIfProjectColour function

    func checkIfProjectColour(colour : Colour) -> Bool {
        if let proCols = project.projectColours {
            for proCol in proCols {
                let proCol = proCol as! ProjectColour
                if let col = proCol.colour {
                    if col == colour {
                        return true
                    }
                }
            }
        }
        return false
    }

and the add/remove functions

    func addProjectColour(colour : Colour) {
        let projectColour = ProjectColour(context: viewContext)
        projectColour.project = project
        projectColour.colour = colour
        
        colour.addToProjectColours(projectColour)
        project.addToProjectColours(projectColour)
        
        do {
            try viewContext.save()
        } catch {
            let nsError = error as NSError
            fatalError("Unresolved error \(nsError), \(nsError.userInfo)")
        }
    }

    func removeProjectColour(colour: Colour) {
        if let proCols = project.projectColours {
            for proCol in proCols {
                let proCol = proCol as! ProjectColour
                if let col = proCol.colour {
                    if col == colour {
                        viewContext.delete(proCol)
                        
                        do {
                            try viewContext.save()
                        } catch {
                            let nsError = error as NSError
                            fatalError("Unresolved error \(nsError), \(nsError.userInfo)")
                        }
                    }
                }
            }
        }
    }