private void doRemoveMarker(MarkerSet markers, Program program) {
		if (markers == null || program == null) {
			return;
		}

		// per-program list
		MarkerSetCacheEntry entry = markerSetCache.get(program);
		if (entry == null) {
			return;
		}
		entry.removeSet(markers);

		// per-group list
		// We need to find the marker by searching through the map of maps (when used in a
		// group setting the marker manager will be mapped directly to one program)
		Collection<Map<Program, MarkerSetImpl>> values = programMarkersByGroup.values();
		for (Map<Program, MarkerSetImpl> map : values) {
			MarkerSetImpl markerSetImpl = map.get(program);
			if (markerSetImpl == markers) {
				map.clear();
				break;
			}
		}

	}
Buffers requests to repaint and notify of marker changes
Ghidra/Features/Base/src/main/java/ghidra/app/plugin/core/marker/MarkerManager.java