{
  "schemaVersion": 2,
  "lastUpdated": "2026-10-24T17:27:00Z",
  "scoring": "overall = average of the functions and queries category pass rates",
  "totalEngines": 2,
  "averagePassRate": 57.9,
  "engines": [
    {
      "rank": 1,
      "schemaVersion": 2,
      "engineName": "DuckDB",
      "engineVersion": "1.5.6",
      "consumer": "substrait community extension",
      "provenance": {
        "source": "ci",
        "runUrl": "https://github.com/IBM/substrait-compliance/actions/runs/40115772370"
      },
      "passRate": 62.0,
      "categories": {
        "functions": {
          "suites": [
            "functions-substrait-0.107"
          ],
          "passed": 808,
          "failed": 355,
          "skipped": 152,
          "passRate": 69.5
        },
        "queries": {
          "suites": [
            "tpch"
          ],
          "passed": 12,
          "failed": 10,
          "skipped": 0,
          "passRate": 54.5
        }
      },
      "passed": 820,
      "failed": 365,
      "skipped": 152,
      "totalTests": 1337,
      "timestamp": "2026-10-24T17:26:02Z",
      "suites": {
        "functions-substrait-0.107": {
          "engineVersion": "1.5.6",
          "totalTests": 1315,
          "passed": 808,
          "failed": 355,
          "skipped": 152,
          "passRate": 69.5,
          "mode": "plan",
          "engineCrashes": 62,
          "extensions": {
            "aggregate_approx": {
              "totalTests": 8,
              "passed": 8,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "aggregate_generic": {
              "totalTests": 5,
              "passed": 5,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 338,
              "passed": 207,
              "failed": 107,
              "skipped": 24,
              "passRate": 65.9
            },
            "arithmetic_decimal": {
              "totalTests": 128,
              "passed": 72,
              "failed": 54,
              "skipped": 2,
              "passRate": 57.1
            },
            "arithmetic_unsigned": {
              "totalTests": 58,
              "passed": 0,
              "failed": 0,
              "skipped": 58,
              "passRate": null
            },
            "boolean": {
              "totalTests": 54,
              "passed": 37,
              "failed": 17,
              "skipped": 0,
              "passRate": 68.5
            },
            "comparison": {
              "totalTests": 183,
              "passed": 171,
              "failed": 12,
              "skipped": 0,
              "passRate": 93.4
            },
            "datetime": {
              "totalTests": 105,
              "passed": 87,
              "failed": 18,
              "skipped": 0,
              "passRate": 82.9
            },
            "list": {
              "totalTests": 45,
              "passed": 0,
              "failed": 6,
              "skipped": 39,
              "passRate": 0.0
            },
            "logarithmic": {
              "totalTests": 41,
              "passed": 19,
              "failed": 4,
              "skipped": 18,
              "passRate": 82.6
            },
            "rounding": {
              "totalTests": 12,
              "passed": 12,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "rounding_decimal": {
              "totalTests": 10,
              "passed": 10,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "string": {
              "totalTests": 322,
              "passed": 180,
              "failed": 131,
              "skipped": 11,
              "passRate": 57.9
            },
            "window": {
              "totalTests": 6,
              "passed": 0,
              "failed": 6,
              "skipped": 0,
              "passRate": 0.0
            }
          },
          "category": "functions"
        },
        "tpch": {
          "engineVersion": "1.5.6",
          "totalTests": 22,
          "passed": 12,
          "failed": 10,
          "skipped": 0,
          "passRate": 54.5,
          "category": "queries"
        },
        "functions-substrait-0.107-sql": {
          "engineVersion": "1.5.6",
          "totalTests": 1315,
          "passed": 1189,
          "failed": 77,
          "skipped": 49,
          "passRate": 93.9,
          "mode": "sql",
          "extensions": {
            "aggregate_approx": {
              "totalTests": 8,
              "passed": 8,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "aggregate_generic": {
              "totalTests": 5,
              "passed": 5,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 338,
              "passed": 313,
              "failed": 5,
              "skipped": 20,
              "passRate": 98.4
            },
            "arithmetic_decimal": {
              "totalTests": 128,
              "passed": 84,
              "failed": 42,
              "skipped": 2,
              "passRate": 66.7
            },
            "arithmetic_unsigned": {
              "totalTests": 58,
              "passed": 50,
              "failed": 1,
              "skipped": 7,
              "passRate": 98.0
            },
            "boolean": {
              "totalTests": 54,
              "passed": 54,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "comparison": {
              "totalTests": 183,
              "passed": 183,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "datetime": {
              "totalTests": 105,
              "passed": 105,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "list": {
              "totalTests": 45,
              "passed": 43,
              "failed": 0,
              "skipped": 2,
              "passRate": 100.0
            },
            "logarithmic": {
              "totalTests": 41,
              "passed": 25,
              "failed": 0,
              "skipped": 16,
              "passRate": 100.0
            },
            "rounding": {
              "totalTests": 12,
              "passed": 12,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "rounding_decimal": {
              "totalTests": 10,
              "passed": 10,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "string": {
              "totalTests": 322,
              "passed": 291,
              "failed": 29,
              "skipped": 2,
              "passRate": 90.9
            },
            "window": {
              "totalTests": 6,
              "passed": 6,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            }
          },
          "informational": true
        },
        "functions-substrait-0.107-extended": {
          "engineVersion": "1.5.6",
          "totalTests": 2278,
          "passed": 1928,
          "failed": 264,
          "skipped": 86,
          "passRate": 88.0,
          "mode": "plan",
          "engineCrashes": 19,
          "extensions": {
            "aggregate_generic": {
              "totalTests": 22,
              "passed": 22,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 973,
              "passed": 767,
              "failed": 142,
              "skipped": 64,
              "passRate": 84.4
            },
            "comparison": {
              "totalTests": 641,
              "passed": 603,
              "failed": 37,
              "skipped": 1,
              "passRate": 94.2
            },
            "datetime": {
              "totalTests": 78,
              "passed": 49,
              "failed": 29,
              "skipped": 0,
              "passRate": 62.8
            },
            "logarithmic": {
              "totalTests": 64,
              "passed": 55,
              "failed": 0,
              "skipped": 9,
              "passRate": 100.0
            },
            "rounding": {
              "totalTests": 93,
              "passed": 93,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "string": {
              "totalTests": 407,
              "passed": 339,
              "failed": 56,
              "skipped": 12,
              "passRate": 85.8
            }
          },
          "informational": true
        },
        "functions-substrait-0.107-extended-sql": {
          "engineVersion": "1.5.6",
          "totalTests": 2278,
          "passed": 2151,
          "failed": 51,
          "skipped": 76,
          "passRate": 97.7,
          "mode": "sql",
          "extensions": {
            "aggregate_generic": {
              "totalTests": 22,
              "passed": 22,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 973,
              "passed": 885,
              "failed": 24,
              "skipped": 64,
              "passRate": 97.4
            },
            "comparison": {
              "totalTests": 641,
              "passed": 615,
              "failed": 25,
              "skipped": 1,
              "passRate": 96.1
            },
            "datetime": {
              "totalTests": 78,
              "passed": 78,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "logarithmic": {
              "totalTests": 64,
              "passed": 55,
              "failed": 0,
              "skipped": 9,
              "passRate": 100.0
            },
            "rounding": {
              "totalTests": 93,
              "passed": 93,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "string": {
              "totalTests": 407,
              "passed": 403,
              "failed": 2,
              "skipped": 2,
              "passRate": 99.5
            }
          },
          "informational": true
        },
        "tpch-substrait-0.81": {
          "engineVersion": "1.5.6",
          "totalTests": 22,
          "passed": 3,
          "failed": 19,
          "skipped": 0,
          "passRate": 13.6,
          "informational": true
        },
        "functions-legacy": {
          "engineVersion": "1.5.6",
          "totalTests": 5037,
          "passed": 3237,
          "failed": 455,
          "skipped": 1345,
          "passRate": 87.7,
          "mode": "sql",
          "extensions": {
            "aggregate": {
              "totalTests": 250,
              "passed": 155,
              "failed": 53,
              "skipped": 42,
              "passRate": 74.5
            },
            "arithmetic": {
              "totalTests": 1615,
              "passed": 1296,
              "failed": 173,
              "skipped": 146,
              "passRate": 88.2
            },
            "array": {
              "totalTests": 154,
              "passed": 0,
              "failed": 0,
              "skipped": 154,
              "passRate": null
            },
            "cast": {
              "totalTests": 78,
              "passed": 66,
              "failed": 2,
              "skipped": 10,
              "passRate": 97.1
            },
            "comparison": {
              "totalTests": 780,
              "passed": 705,
              "failed": 53,
              "skipped": 22,
              "passRate": 93.0
            },
            "conditional": {
              "totalTests": 93,
              "passed": 70,
              "failed": 5,
              "skipped": 18,
              "passRate": 93.3
            },
            "datetime": {
              "totalTests": 489,
              "passed": 299,
              "failed": 48,
              "skipped": 142,
              "passRate": 86.2
            },
            "geospatial": {
              "totalTests": 103,
              "passed": 30,
              "failed": 3,
              "skipped": 70,
              "passRate": 90.9
            },
            "json": {
              "totalTests": 86,
              "passed": 30,
              "failed": 16,
              "skipped": 40,
              "passRate": 65.2
            },
            "map": {
              "totalTests": 92,
              "passed": 0,
              "failed": 0,
              "skipped": 92,
              "passRate": null
            },
            "set": {
              "totalTests": 105,
              "passed": 0,
              "failed": 0,
              "skipped": 105,
              "passRate": null
            },
            "string": {
              "totalTests": 900,
              "passed": 551,
              "failed": 98,
              "skipped": 251,
              "passRate": 84.9
            },
            "struct": {
              "totalTests": 59,
              "passed": 35,
              "failed": 4,
              "skipped": 20,
              "passRate": 89.7
            },
            "window": {
              "totalTests": 233,
              "passed": 0,
              "failed": 0,
              "skipped": 233,
              "passRate": null
            }
          },
          "informational": true
        }
      },
      "page": "engines/duckdb.html",
      "findings": {
        "consumer": {
          "groups": 13,
          "cases": 495
        },
        "engine": {
          "groups": 9,
          "cases": 128
        },
        "queries": {
          "groups": 1,
          "cases": 10
        },
        "untested": {
          "groups": 5,
          "cases": 84
        },
        "options": {
          "groups": 32,
          "cases": 158
        }
      }
    },
    {
      "rank": 2,
      "schemaVersion": 2,
      "engineName": "DataFusion",
      "engineVersion": "55.0.0",
      "consumer": "datafusion.substrait 55.0.0",
      "provenance": {
        "source": "ci",
        "runUrl": "https://github.com/IBM/substrait-compliance/actions/runs/40115772344"
      },
      "passRate": 53.8,
      "categories": {
        "functions": {
          "suites": [
            "functions-substrait-0.107"
          ],
          "passed": 820,
          "failed": 332,
          "skipped": 163,
          "passRate": 71.2
        },
        "queries": {
          "suites": [
            "tpch"
          ],
          "passed": 8,
          "failed": 14,
          "skipped": 0,
          "passRate": 36.4
        }
      },
      "passed": 828,
      "failed": 346,
      "skipped": 163,
      "totalTests": 1337,
      "timestamp": "2026-10-24T17:22:37Z",
      "suites": {
        "functions-substrait-0.107": {
          "engineVersion": "55.0.0",
          "totalTests": 1315,
          "passed": 820,
          "failed": 332,
          "skipped": 163,
          "passRate": 71.2,
          "mode": "plan",
          "extensions": {
            "aggregate_approx": {
              "totalTests": 8,
              "passed": 0,
              "failed": 8,
              "skipped": 0,
              "passRate": 0.0
            },
            "aggregate_generic": {
              "totalTests": 5,
              "passed": 5,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 338,
              "passed": 205,
              "failed": 97,
              "skipped": 36,
              "passRate": 67.9
            },
            "arithmetic_decimal": {
              "totalTests": 128,
              "passed": 76,
              "failed": 49,
              "skipped": 3,
              "passRate": 60.8
            },
            "arithmetic_unsigned": {
              "totalTests": 58,
              "passed": 0,
              "failed": 0,
              "skipped": 58,
              "passRate": null
            },
            "boolean": {
              "totalTests": 54,
              "passed": 54,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "comparison": {
              "totalTests": 183,
              "passed": 171,
              "failed": 12,
              "skipped": 0,
              "passRate": 93.4
            },
            "datetime": {
              "totalTests": 105,
              "passed": 77,
              "failed": 28,
              "skipped": 0,
              "passRate": 73.3
            },
            "list": {
              "totalTests": 45,
              "passed": 6,
              "failed": 0,
              "skipped": 39,
              "passRate": 100.0
            },
            "logarithmic": {
              "totalTests": 41,
              "passed": 25,
              "failed": 0,
              "skipped": 16,
              "passRate": 100.0
            },
            "rounding": {
              "totalTests": 12,
              "passed": 12,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "rounding_decimal": {
              "totalTests": 10,
              "passed": 9,
              "failed": 1,
              "skipped": 0,
              "passRate": 90.0
            },
            "string": {
              "totalTests": 322,
              "passed": 174,
              "failed": 137,
              "skipped": 11,
              "passRate": 55.9
            },
            "window": {
              "totalTests": 6,
              "passed": 6,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            }
          },
          "category": "functions"
        },
        "tpch": {
          "engineVersion": "55.0.0",
          "totalTests": 22,
          "passed": 8,
          "failed": 14,
          "skipped": 0,
          "passRate": 36.4,
          "category": "queries"
        },
        "functions-substrait-0.107-sql": {
          "engineVersion": "55.0.0",
          "totalTests": 1315,
          "passed": 1089,
          "failed": 80,
          "skipped": 146,
          "passRate": 93.2,
          "mode": "sql",
          "extensions": {
            "aggregate_approx": {
              "totalTests": 8,
              "passed": 8,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "aggregate_generic": {
              "totalTests": 5,
              "passed": 5,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 338,
              "passed": 291,
              "failed": 0,
              "skipped": 47,
              "passRate": 100.0
            },
            "arithmetic_decimal": {
              "totalTests": 128,
              "passed": 74,
              "failed": 51,
              "skipped": 3,
              "passRate": 59.2
            },
            "arithmetic_unsigned": {
              "totalTests": 58,
              "passed": 38,
              "failed": 0,
              "skipped": 20,
              "passRate": 100.0
            },
            "boolean": {
              "totalTests": 54,
              "passed": 54,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "comparison": {
              "totalTests": 183,
              "passed": 171,
              "failed": 0,
              "skipped": 12,
              "passRate": 100.0
            },
            "datetime": {
              "totalTests": 105,
              "passed": 105,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "list": {
              "totalTests": 45,
              "passed": 18,
              "failed": 0,
              "skipped": 27,
              "passRate": 100.0
            },
            "logarithmic": {
              "totalTests": 41,
              "passed": 25,
              "failed": 0,
              "skipped": 16,
              "passRate": 100.0
            },
            "rounding": {
              "totalTests": 12,
              "passed": 12,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "rounding_decimal": {
              "totalTests": 10,
              "passed": 9,
              "failed": 1,
              "skipped": 0,
              "passRate": 90.0
            },
            "string": {
              "totalTests": 322,
              "passed": 273,
              "failed": 28,
              "skipped": 21,
              "passRate": 90.7
            },
            "window": {
              "totalTests": 6,
              "passed": 6,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            }
          },
          "informational": true
        },
        "functions-substrait-0.107-extended": {
          "engineVersion": "55.0.0",
          "totalTests": 2278,
          "passed": 1889,
          "failed": 320,
          "skipped": 69,
          "passRate": 85.5,
          "mode": "plan",
          "extensions": {
            "aggregate_generic": {
              "totalTests": 22,
              "passed": 22,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 973,
              "passed": 778,
              "failed": 140,
              "skipped": 55,
              "passRate": 84.7
            },
            "comparison": {
              "totalTests": 641,
              "passed": 583,
              "failed": 57,
              "skipped": 1,
              "passRate": 91.1
            },
            "datetime": {
              "totalTests": 78,
              "passed": 31,
              "failed": 47,
              "skipped": 0,
              "passRate": 39.7
            },
            "logarithmic": {
              "totalTests": 64,
              "passed": 64,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "rounding": {
              "totalTests": 93,
              "passed": 92,
              "failed": 1,
              "skipped": 0,
              "passRate": 98.9
            },
            "string": {
              "totalTests": 407,
              "passed": 319,
              "failed": 75,
              "skipped": 13,
              "passRate": 81.0
            }
          },
          "informational": true
        },
        "functions-substrait-0.107-extended-sql": {
          "engineVersion": "55.0.0",
          "totalTests": 2278,
          "passed": 2101,
          "failed": 38,
          "skipped": 139,
          "passRate": 98.2,
          "mode": "sql",
          "extensions": {
            "aggregate_generic": {
              "totalTests": 22,
              "passed": 22,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "arithmetic": {
              "totalTests": 973,
              "passed": 880,
              "failed": 5,
              "skipped": 88,
              "passRate": 99.4
            },
            "comparison": {
              "totalTests": 641,
              "passed": 587,
              "failed": 25,
              "skipped": 29,
              "passRate": 95.9
            },
            "datetime": {
              "totalTests": 78,
              "passed": 78,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "logarithmic": {
              "totalTests": 64,
              "passed": 64,
              "failed": 0,
              "skipped": 0,
              "passRate": 100.0
            },
            "rounding": {
              "totalTests": 93,
              "passed": 92,
              "failed": 1,
              "skipped": 0,
              "passRate": 98.9
            },
            "string": {
              "totalTests": 407,
              "passed": 378,
              "failed": 7,
              "skipped": 22,
              "passRate": 98.2
            }
          },
          "informational": true
        },
        "tpch-substrait-0.81": {
          "engineVersion": "55.0.0",
          "totalTests": 22,
          "passed": 8,
          "failed": 14,
          "skipped": 0,
          "passRate": 36.4,
          "informational": true
        }
      },
      "page": "engines/datafusion.html",
      "findings": {
        "consumer": {
          "groups": 8,
          "cases": 536
        },
        "engine": {
          "groups": 12,
          "cases": 118
        },
        "queries": {
          "groups": 2,
          "cases": 14
        },
        "untested": {
          "groups": 12,
          "cases": 238
        },
        "options": {
          "groups": 26,
          "cases": 135
        }
      }
    }
  ],
  "unranked": []
}