protected TransformerInput buildTransformerInput(List<Long> tokens, int maxTokens, boolean isQuery) {
        if (!isQuery) {
            tokens = tokens.stream().filter(token -> !skipTokens.contains(token)).toList();
        }
        List<Long> inputIds = new ArrayList<>(maxTokens);
        List<Long> attentionMask = new ArrayList<>(maxTokens);
        if (tokens.size() > maxTokens - 3)
            tokens = tokens.subList(0, maxTokens - 3);
        inputIds.add(startSequenceToken);
        inputIds.add(isQuery? querySequenceToken: documentSequenceToken);
        inputIds.addAll(tokens);
        inputIds.add(endSequenceToken);

        int inputLength = inputIds.size();
        long padTokenId = isQuery? maskSequenceToken: padSequenceToken;

        int padding = isQuery? maxTokens - inputLength: 0;
        for (int i = 0; i < padding; i++)
            inputIds.add(padTokenId);

        for (int i = 0; i < inputLength; i++)
            attentionMask.add((long) 1);

        for (int i = 0; i < padding; i++)
            attentionMask.add((long) 0); // Do not attend to mask paddings

        return new TransformerInput(inputIds, attentionMask);
    }
@Test
    public void testInputTensorsWordPiece() {
        // wordPiece tokenizer("this is a query !") -> [2023, 2003, 1037, 23032, 999]
        List<Long> tokens = List.of(2023L, 2003L, 1037L, 23032L, 999L);
        ColBertEmbedder.TransformerInput input = embedder.buildTransformerInput(tokens,10,true);
        assertEquals(10,input.inputIds().size());
        assertEquals(10,input.attentionMask().size());
        assertEquals(List.of(101L, 1L, 2023L, 2003L, 1037L, 23032L, 999L, 102L, 103L, 103L),input.inputIds());
        assertEquals(List.of(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 0L, 0L),input.attentionMask());

        input = embedder.buildTransformerInput(tokens,10,false);
        assertEquals(7,input.inputIds().size());
        assertEquals(7,input.attentionMask().size());
        assertEquals(List.of(101L, 2L, 2023L, 2003L, 1037L, 23032L, 102L),input.inputIds());
        assertEquals(List.of(1L, 1L, 1L, 1L, 1L, 1L, 1L),input.attentionMask());
    }