private void subsimplify(CoordinateSequence in, MutableCoordinateSequence out, int first, int last,
    int numForcedPoints) {
    // numForcePoints lets us keep some points even if they are below simplification threshold
    boolean force = numForcedPoints > 0;
    double maxSqDist = force ? -1 : sqTolerance;
    int index = -1;
    double p1x = in.getX(first);
    double p1y = in.getY(first);
    double p2x = in.getX(last);
    double p2y = in.getY(last);

    for (int i = first + 1; i < last; i++) {
      double px = in.getX(i);
      double py = in.getY(i);
      double sqDist = getSqSegDist(px, py, p1x, p1y, p2x, p2y);

      if (sqDist > maxSqDist) {
        index = i;
        maxSqDist = sqDist;
      }
    }

    if (force || maxSqDist > sqTolerance) {
      if (index - first > 1) {
        subsimplify(in, out, first, index, numForcedPoints - 1);
      }
      out.forceAddPoint(in.getX(index), in.getY(index));
      if (last - index > 1) {
        subsimplify(in, out, index, last, numForcedPoints - 2);
      }
    }
  }
Returns the square of the number of units that (px, p1) is away from the line segment from (p1x, py1) to (p2x,
p2y).
planetiler-core/src/main/java/com/onthegomap/planetiler/geo/DouglasPeuckerSimplifier.java