private void subsimplify(List<Coordinate> in, List<Coordinate> out, int first, int last, int numForcedPoints) {
    // numForcePoints lets us keep some points even if they are below simplification threshold
    boolean force = numForcedPoints > 0;
    double maxSqDist = force ? -1 : sqTolerance;
    int index = -1;
    Coordinate p1 = in.get(first);
    Coordinate p2 = in.get(last);
    double p1x = p1.x;
    double p1y = p1.y;
    double p2x = p2.x;
    double p2y = p2.y;

    int i = first + 1;
    Coordinate furthest = null;
    for (Coordinate coord : in.subList(first + 1, last)) {
      double sqDist = getSqSegDist(coord.x, coord.y, p1x, p1y, p2x, p2y);

      if (sqDist > maxSqDist) {
        index = i;
        furthest = coord;
        maxSqDist = sqDist;
      }
      i++;
    }

    if (force || maxSqDist > sqTolerance) {
      if (index - first > 1) {
        subsimplify(in, out, first, index, numForcedPoints - 1);
      }
      out.add(furthest);
      if (last - index > 1) {
        subsimplify(in, out, index, last, numForcedPoints - 2);
      }
    }
  }
Returns the square of the number of units that (px, p1) is away from the line segment from (p1x, py1) to (p2x,
p2y).
planetiler-core/src/main/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifier.java