@Override
    public synchronized Request poll(Task task) {
		try (Jedis jedis = pool.getResource()) {
            String url = jedis.lpop(getQueueKey(task));
            if (url == null) {
                return null;
            }
            String key = ITEM_PREFIX + task.getUUID();
            String field = DigestUtils.sha1Hex(url);
            byte[] bytes = jedis.hget(key.getBytes(), field.getBytes());
            if (bytes != null) {
                Request o = JSON.parseObject(new String(bytes), Request.class);
                return o;
            }
            Request request = new Request(url);
            return request;
        }
    }
Use Redis as url scheduler for distributed crawlers.<br>

@author code4crafter@gmail.com <br>
@since 0.2.0
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java