Studying Box2D-Lite in Javascript, Part 10.3: Testing Polygon Convexity

  /// This determines if a hull is valid. Checks for:
    /// - convexity
    /// - collinear points
    /// This is expensive and should not be called at runtime.

    static b2ValidateHull(hull) {

        if (hull.count < 3 || b2_maxPolygonVertices < hull.count) {
            return false;
        }

        // test that every point is behind every edge
        for (let i = 0; i < hull.count; i++)	{
            // create an edge vector
            let i1 = i;
            let i2 = i < hull.count - 1 ? i1 + 1 : 0;
            let p = hull.points[i1];
            let e = Vec2.Subtract(hull.points[i2], p);
            e.normalize();

            for (let j = 0; j < hull.count; j++)		{
                // skip points that subtend the current edge
                if (j == i1 || j == i2) {
                    continue;
                }

                let distance = b2Cross(Vec2.Subtract(hull.points[j], p), e);
                if (distance >= 0.0) {
                    return false;
                }
            }
        }

        // test for collinear points
        for (let i = 0; i < hull.count; i++)	{
            let i1 = i;
            let i2 = (i + 1) % hull.count;
            let i3 = (i + 2) % hull.count;

            let p1 = hull.points[i1];
            let p2 = hull.points[i2];
            let p3 = hull.points[i3];

            let e = Vec2.Subtract(p3, p1);
            e.normalize();

            let v = Vec2.Subtract(p2, p1);
            let distance = b2Cross(Vec2.Subtract(p2, p1), e);
            if (distance <= b2_linearSlop) {
                // p1-p2-p3 are collinear
                return false;
            }
        }

        return true;
    };