protected TransformerInput buildTransformerInput(List<Long> tokens, int maxTokens, boolean isQuery) {
        if (!isQuery) {
            tokens = tokens.stream().filter(token -> !skipTokens.contains(token)).toList();
        }
        List<Long> inputIds = new ArrayList<>(maxTokens);
        List<Long> attentionMask = new ArrayList<>(maxTokens);
        if (tokens.size() > maxTokens - 3)
            tokens = tokens.subList(0, maxTokens - 3);
        inputIds.add(startSequenceToken);
        inputIds.add(isQuery? querySequenceToken: documentSequenceToken);
        inputIds.addAll(tokens);
        inputIds.add(endSequenceToken);

        int inputLength = inputIds.size();
        long padTokenId = isQuery? maskSequenceToken: padSequenceToken;

        int padding = isQuery? maxTokens - inputLength: 0;
        for (int i = 0; i < padding; i++)
            inputIds.add(padTokenId);

        for (int i = 0; i < inputLength; i++)
            attentionMask.add((long) 1);

        for (int i = 0; i < padding; i++)
            attentionMask.add((long) 0); // Do not attend to mask paddings

        return new TransformerInput(inputIds, attentionMask);
    }
@Test
    public void testInputTensorsSentencePiece() {
        // Sentencepiece tokenizer("this is a query !") -> [903, 83, 10, 41, 1294, 711]
        // ! is mapped to 711 and is a punctuation character
        List<Long> tokens = List.of(903L, 83L, 10L, 41L, 1294L, 711L);
        ColBertEmbedder.TransformerInput input = multiLingualEmbedder.buildTransformerInput(tokens,10,true);
        assertEquals(10,input.inputIds().size());
        assertEquals(10,input.attentionMask().size());
        assertEquals(List.of(0L, 3L, 903L, 83L, 10L, 41L, 1294L, 711L, 2L, 250001L),input.inputIds());
        assertEquals(List.of(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 0L),input.attentionMask());

        // NO padding for document side and 711 (punctuation) is now filtered out
        input = multiLingualEmbedder.buildTransformerInput(tokens,10,false);
        assertEquals(8,input.inputIds().size());
        assertEquals(8,input.attentionMask().size());
        assertEquals(List.of(0L, 4L, 903L, 83L, 10L, 41L, 1294L, 2L),input.inputIds());
        assertEquals(List.of(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L),input.attentionMask());

        input = multiLingualEmbedder.buildTransformerInput(List.of(711L), 5, true);
        assertEquals(List.of(0L, 3L, 711L,2L, 250001L),input.inputIds());
        assertEquals(List.of(1L, 1L, 1L, 1L, 0L),input.attentionMask());

        input = multiLingualEmbedder.buildTransformerInput(List.of(711L), 5, false);
        assertEquals(List.of(0L, 4L, 2L),input.inputIds());
        assertEquals(List.of(1L, 1L, 1L),input.attentionMask());
    }