From 76a46ecbb7995ee3a622450a95a560fcabe05423 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 24 Oct 2026 10:34:27 +0900 Subject: [PATCH 1/2] Port the JRE module trim and the jar closure from humemai/arcadedb-embedded-python Ported from humemai/arcadedb-embedded-python#409 (bc93f97a67) and #452 (aebf4373b5): - scripts/jre_module_exclusions.txt lists the JDK modules the bundled JRE must not contain (java.desktop, jdk.jfr, java.compiler). Dockerfile.build and build-native.sh remove them from the jdeps answer before jlink, stop when jdeps finds nothing (the java.se fallback would bring them back), and fail when one is still in `java --list-modules`. The Linux build also strips the runtime's .so files. - scripts/jar_exclusions.txt no longer excludes jackson, snappy-java and commons-math3: server mode (openapi.json, snappy bodies) and jvector refer to them. - tests/test_jar_closure.py reads the class files of the shipped jars and fails on a reference into a package no shipped jar or JDK module provides, unless listed with a reason. - tests/test_server_packaging.py gains the openapi.json and snappy body tests. Refs #19 Co-Authored-By: Claude Opus 5.5 --- scripts/Dockerfile.build | 22 +- scripts/build-native.sh | 31 +- scripts/jar_exclusions.txt | 15 +- scripts/jre_module_exclusions.txt | 24 ++ tests/test_jar_closure.py | 535 ++++++++++++++++++++++++++++++ tests/test_server_packaging.py | 99 ++++++ 6 files changed, 711 insertions(+), 15 deletions(-) create mode 100644 scripts/jre_module_exclusions.txt create mode 100644 tests/test_jar_closure.py diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index d61d758..82ca788 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -69,6 +69,8 @@ RUN if [ "$USE_LOCAL_JARS" = "1" ]; then \ # Copy JAR exclusion list COPY scripts/jar_exclusions.txt /build/jar_exclusions.txt +# JDK modules the bundled JRE must not contain (see the file for why) +COPY scripts/jre_module_exclusions.txt /build/jre_module_exclusions.txt # Remove excluded JARs based on jar_exclusions.txt RUN echo "🗑️ Removing excluded JARs..." && \ @@ -107,11 +109,14 @@ RUN echo "🔍 Analyzing JARs with jdeps..." && \ else \ FILTERED_MODULES="${DETECTED_MODULES}" ; \ fi && \ - if [ -n "${FILTERED_MODULES}" ]; then \ - REQUIRED_MODULES="${FILTERED_MODULES},jdk.management,jdk.zipfs,jdk.unsupported,jdk.incubator.vector" ; \ - else \ - REQUIRED_MODULES="java.se,jdk.management,jdk.zipfs,jdk.unsupported,jdk.incubator.vector" ; \ + grep -v '^#' /build/jre_module_exclusions.txt | tr -d '\r' | sed '/^[[:space:]]*$/d' > /build/jre_excluded_modules.list && \ + echo "📦 Excluded modules: $(paste -sd ',' /build/jre_excluded_modules.list)" && \ + FILTERED_MODULES=$(echo "${FILTERED_MODULES}" | tr ',' '\n' | sed '/^$/d' | grep -vxFf /build/jre_excluded_modules.list | paste -sd "," -) && \ + if [ -z "${FILTERED_MODULES}" ]; then \ + echo "❌ jdeps found no modules; the java.se fallback would bring back the excluded modules, so the build stops" && \ + exit 1 ; \ fi && \ + REQUIRED_MODULES="${FILTERED_MODULES},jdk.management,jdk.zipfs,jdk.unsupported,jdk.incubator.vector" && \ echo "🔨 Building minimal JRE for platform: ${TARGET_PLATFORM}" && \ echo "📦 Detected modules (raw): ${DETECTED_MODULES}" && \ echo "📦 Detected modules (filtered): ${FILTERED_MODULES}" && \ @@ -135,6 +140,15 @@ RUN echo "🔍 Analyzing JARs with jdeps..." && \ fi ; \ echo "" ; \ echo "✅ JRE build complete!" ; \ + echo "📊 JRE size after jlink: $(du -sm /build/jre | cut -f1) MB" ; \ + BAD_MODULES=$(/build/jre/bin/java --list-modules | sed 's/@.*//' | grep -xFf /build/jre_excluded_modules.list || true) ; \ + if [ -n "$BAD_MODULES" ]; then \ + echo "❌ the JRE contains excluded modules (another module requires them): $BAD_MODULES" ; \ + exit 1 ; \ + fi ; \ + echo "✅ No excluded module in the JRE" ; \ + find /build/jre -name '*.so' -type f -exec strip --strip-unneeded {} + ; \ + echo "📊 JRE size after stripping native libraries: $(du -sm /build/jre | cut -f1) MB" ; \ else \ echo "⚠️ jmods directory not found under ${JAVA_HOME}. Falling back to copying full JDK runtime." ; \ mkdir -p /build/jre ; \ diff --git a/scripts/build-native.sh b/scripts/build-native.sh index 9957f95..d658567 100755 --- a/scripts/build-native.sh +++ b/scripts/build-native.sh @@ -21,6 +21,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PY_BINDINGS_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" DIST_DIR="$PY_BINDINGS_DIR/dist" JAR_EXCLUSIONS_FILE="$SCRIPT_DIR/jar_exclusions.txt" +JRE_MODULE_EXCLUSIONS_FILE="$SCRIPT_DIR/jre_module_exclusions.txt" # Parameters PLATFORM="${1:-}" @@ -238,13 +239,23 @@ fi # jdk.incubator.vector: Required for vectorized execution paths # NOTE: the java.se umbrella used to be force-added as a safety net; with the # package embedded-only and jdeps detection fixed, the detected set gates the -# JRE (full suite + examples verify each build). java.se remains the fallback -# when detection yields nothing. -if [ -n "$FILTERED_MODULES" ]; then - REQUIRED_MODULES="${FILTERED_MODULES},jdk.management,jdk.zipfs,jdk.unsupported,jdk.incubator.vector" -else - REQUIRED_MODULES="java.se,jdk.management,jdk.zipfs,jdk.unsupported,jdk.incubator.vector" +# JRE (full suite + examples verify each build). Detection that yields nothing +# stops the build: the java.se umbrella would bring back the excluded modules. +# JDK modules the bundled JRE must not contain (jre_module_exclusions.txt says why). +# Removed from the detected set here, and checked in the finished JRE below. +if [[ ! -f "$JRE_MODULE_EXCLUSIONS_FILE" ]]; then + echo -e "${RED}❌ JRE module exclusion file not found: ${JRE_MODULE_EXCLUSIONS_FILE}${NC}" + exit 1 +fi +EXCLUDED_MODULES_LIST="$(mktemp)" +grep -v '^#' "$JRE_MODULE_EXCLUSIONS_FILE" | tr -d '\r' | sed '/^[[:space:]]*$/d' > "$EXCLUDED_MODULES_LIST" +echo -e "${CYAN}📦 Excluded modules: ${YELLOW}$(paste -sd ',' "$EXCLUDED_MODULES_LIST")${NC}" +FILTERED_MODULES=$(echo "$FILTERED_MODULES" | tr ',' '\n' | sed '/^$/d' | { grep -vxFf "$EXCLUDED_MODULES_LIST" || true; } | paste -sd "," -) +if [ -z "$FILTERED_MODULES" ]; then + echo -e "${RED}❌ jdeps found no modules; the java.se fallback would bring back the excluded modules, so the build stops${NC}" + exit 1 fi +REQUIRED_MODULES="${FILTERED_MODULES},jdk.management,jdk.zipfs,jdk.unsupported,jdk.incubator.vector" echo -e "${CYAN}📦 Detected modules (raw): ${YELLOW}${DETECTED_MODULES}${NC}" echo -e "${CYAN}📦 Detected modules (filtered): ${YELLOW}${FILTERED_MODULES}${NC}" @@ -262,6 +273,14 @@ jlink \ --output "$PY_BINDINGS_DIR/temp_jre" echo -e "${GREEN}✅ JRE built${NC}" +# A module that another module requires comes back through jlink's own resolution. +BAD_MODULES=$("$PY_BINDINGS_DIR/temp_jre/bin/java" --list-modules | tr -d '\r' | sed 's/@.*//' | { grep -xFf "$EXCLUDED_MODULES_LIST" || true; }) +rm -f "$EXCLUDED_MODULES_LIST" +if [ -n "$BAD_MODULES" ]; then + echo -e "${RED}❌ the JRE contains excluded modules (another module requires them): ${BAD_MODULES}${NC}" + exit 1 +fi +echo -e "${GREEN}✅ No excluded module in the JRE${NC}" JRE_SIZE=$(du -sh "$PY_BINDINGS_DIR/temp_jre" | cut -f1) echo -e "${CYAN}📊 JRE size: ${YELLOW}${JRE_SIZE}${NC}" diff --git a/scripts/jar_exclusions.txt b/scripts/jar_exclusions.txt index 16fa764..d875247 100644 --- a/scripts/jar_exclusions.txt +++ b/scripts/jar_exclusions.txt @@ -18,10 +18,15 @@ regex-*.jar arcadedb-ha-raft-*.jar arcadedb-metrics-*.jar arcadedb-mongodbw-*.jar -# Embedded-unused (verified 2026-07-05 by quarantine + full suite/examples gate; -# micrometer must STAY - server startup requires it): -commons-math3-*.jar -jackson-*.jar -snappy-java-*.jar +# Verified 2026-07-05 by quarantine + full suite/examples gate; micrometer must STAY +# (server startup requires it): arcadedb-tracing-*.jar jline-*.jar +# +# NOT excluded, on purpose: that gate ran the embedded suite only, and these are +# reached in server mode (tests/test_jar_closure.py finds this class of mistake): +# jackson-*.jar swagger-core builds GET /api/v1/openapi.json with core, +# annotations, databind, dataformat-yaml and datatype-jsr310 +# snappy-java-*.jar CompressedBodyDecoder and the Prometheus remote read and +# write routes +# commons-math3-*.jar jvector's ScoreTracker$TwoPhaseTracker diff --git a/scripts/jre_module_exclusions.txt b/scripts/jre_module_exclusions.txt new file mode 100644 index 0000000..11f6f3d --- /dev/null +++ b/scripts/jre_module_exclusions.txt @@ -0,0 +1,24 @@ +# JDK modules the bundled jlink runtime must NOT contain, one per line. +# +# The module list of the runtime comes from `jdeps --print-module-deps` over the +# shipped jars. jdeps names a module as soon as one class in one jar refers to it, +# so optional users pull in big modules that no code path of the wheel reaches. +# Both build scripts (Dockerfile.build for Linux, build-native.sh for macOS and +# Windows) remove the modules below from that list, and after jlink they fail the +# build when one of them is still in `java --list-modules` (a module that another +# module requires comes back that way). Removing java.desktop also removes +# java.datatransfer and java.prefs, which only it requires. +# +# tests/test_jar_closure.py reads the class files of the shipped jars and fails +# when a class refers to a package of a JDK module that the runtime lacks, unless +# that reference is listed there with a reason. Adding a module here means adding +# the references of the jars it breaks to that list, or keeping the module. +# +# java.desktop: java.beans in Jackson (Java7SupportImpl), snakeyaml, commons-lang3, +# Micrometer Log4j2Metrics; java.awt in JTS (org.locationtech.jts.awt) and +# commons-math3; java.awt.datatransfer in jakarta.activation. About 12 MB of the wheel. +# jdk.jfr: only com.arcadedb.utility.JVMGCHistogramReporter, which nothing calls. +# java.compiler: only org.agrona.generation (agrona's code generator), which nothing calls. +java.desktop +jdk.jfr +java.compiler diff --git a/tests/test_jar_closure.py b/tests/test_jar_closure.py new file mode 100644 index 0000000..16f8f44 --- /dev/null +++ b/tests/test_jar_closure.py @@ -0,0 +1,535 @@ +"""No shipped class points into a package that no shipped jar provides. + +``scripts/jar_exclusions.txt`` removes jars from the wheel. Nothing used to check +that what remains still has what it calls. Three times that gap shipped a +working-looking wheel with a broken feature: + +* Jackson was excluded as "embedded-unused" (2026-07-05), then server mode came + back (2026-08-01) and ``GET /api/v1/openapi.json`` answered 500 + ``NoClassDefFoundError``. +* snappy-java was excluded the same day, and the Prometheus remote read and + write routes, and every snappy-compressed request body, answered 500. +* commons-math3 was excluded while jvector's ``ScoreTracker$TwoPhaseTracker`` + still refers to it. + +An exclusion run (a quarantine plus the test suite) cannot find these, because +the suite only exercises the paths the suite has. This test reads the class files +instead. For every class in every shipped jar it takes the classes the constant +pool names and requires that the package is provided by a shipped jar or by the +JDK. A package that nothing provides must be in ``ALLOWED_DANGLING`` with a +reason. + +The same gap exists one level down: the bundled JRE is a jlink image without the +modules in ``scripts/jre_module_exclusions.txt`` (``java.desktop``, ``jdk.jfr``, +``java.compiler``). A shipped class that names a package of one of those modules +fails with ``NoClassDefFoundError`` when its code path runs, and jdeps cannot say +so, because the build removes those modules from jdeps's answer. The second half of +this file reads the class files against the packages the running JVM provides. + +If this test fails after you excluded a jar, the jar is needed: take it off +``jar_exclusions.txt``. If it fails after an engine or dependency update, a new +class refers to a library the wheel does not ship: decide whether the code path +is reachable, and either ship the jar or add an entry below that says why not. +""" + +import os +import shutil +import struct +import zipfile + +import pytest +from arcadedb_embedded.jvm import get_jar_path + +# Packages the JDK provides. The first test checks the jars only. The bundled JRE +# is a jlink image, and the modules it lacks on purpose are checked by the tests +# at the end of this file (REMOVED_MODULE_PACKAGES); any other module that is +# missing is caught by jdeps at build time, which derives the module list from +# these same jars. +JDK_PREFIXES = ( + "java/", + "javax/", + "jdk/", + "sun/", + "com/sun/", + "org/w3c/", + "org/xml/", + "org/ietf/", + "org/omg/", +) + +# (referencing jar name prefix, package prefix, why a missing package is fine). +# A reference is excused only when the referencing jar AND the package match, so +# a new dangling reference from the engine itself is never excused by an entry +# written for a third-party library. +ALLOWED_DANGLING = ( + ( + "micrometer-", + "", + "Micrometer binders for libraries (Jetty, Kafka, Hibernate, Netty, Log4j, " + "OkHttp, jOOQ, ...) that are loaded only when the application uses that " + "library. ArcadeDB's server registers none of them.", + ), + ( + "jboss-logging", + "org/apache/log4j", + "Optional logging backend; the wheel logs through java.util.logging.", + ), + ( + "jboss-logging", + "org/apache/logging/log4j", + "Optional logging backend; the wheel logs through java.util.logging.", + ), + ( + "jboss-logging", + "org/jboss/logmanager", + "Optional logging backend; the wheel logs through java.util.logging.", + ), + ( + "xnio-", + "org/jboss/logging/annotations", + "Annotations of the JBoss message-logger annotation processor, " + "needed at compile time only.", + ), + ( + "xnio-", + "org/osgi/framework", + "OSGi bundle activator, loaded only inside an OSGi container.", + ), + ( + "snappy-java-", + "org/osgi/framework", + "OSGi bundle activator, loaded only inside an OSGi container.", + ), + ( + "smallrye-common-annotation-", + "jakarta/enterprise/util", + "CDI types for a CDI container; the server is not one.", + ), + ( + "spatial4j-", + "org/noggit", + "Optional GeoJSON reader; ArcadeDB's geo functions do not use it.", + ), + ( + "arcadedb-console-", + "org/jline", + "The interactive console is a command-line tool that is not started from " + "Python; jline is in jar_exclusions.txt for that reason.", + ), +) + + +def _class_references(data): + """The internal names of the classes a class file's constant pool names. + + Parses the constant pool only (JVMS 4.4): the table sits first in the file + and its Class entries (tag 7) are every class the code refers to by name. + """ + count = struct.unpack_from(">H", data, 8)[0] + pos = 10 + utf8 = {} + class_indexes = [] + index = 1 + while index < count: + tag = data[pos] + if tag == 1: + length = struct.unpack_from(">H", data, pos + 1)[0] + utf8[index] = data[pos + 3 : pos + 3 + length] + pos += 3 + length + elif tag == 7: + class_indexes.append(struct.unpack_from(">H", data, pos + 1)[0]) + pos += 3 + elif tag in (8, 16, 19, 20): # String, MethodType, Module, Package + pos += 3 + elif tag in ( + 3, + 4, + 9, + 10, + 11, + 12, + 17, + 18, + ): # int, float, refs, NameAndType, dynamic + pos += 5 + elif tag == 15: # MethodHandle + pos += 4 + elif tag in (5, 6): # long, double take two slots + pos += 9 + index += 1 + else: + raise ValueError(f"unknown constant pool tag {tag}") + index += 1 + return [utf8[i].decode("utf-8", "replace") for i in class_indexes if i in utf8] + + +def _package_of(class_name): + """The package of a Class entry, which is a class name or an array descriptor.""" + name = class_name.lstrip("[") + if name.startswith("L") and name.endswith(";"): + name = name[1:-1] + return name.rsplit("/", 1)[0] if "/" in name else "" + + +def _scan(jar_dir): + """Return ``{(jar name, package): example class}`` for every dangling package.""" + jar_names = sorted(n for n in os.listdir(jar_dir) if n.endswith(".jar")) + provided = set() + references = {} + for jar in jar_names: + with zipfile.ZipFile(os.path.join(jar_dir, jar)) as archive: + classes = [ + n + for n in archive.namelist() + if n.endswith(".class") and not n.startswith("META-INF/") + ] + for name in classes: + provided.add(name.rsplit("/", 1)[0] if "/" in name else "") + for name in classes: + for ref in _class_references(archive.read(name)): + package = _package_of(ref) + if package: + references.setdefault((jar, package), name) + return { + key: example + for key, example in references.items() + if key[1] not in provided and not key[1].startswith(JDK_PREFIXES) + } + + +def _is_allowed(jar, package): + return any( + jar.startswith(jar_prefix) and package.startswith(package_prefix) + for jar_prefix, package_prefix, _ in ALLOWED_DANGLING + ) + + +def test_every_class_reference_resolves_to_a_shipped_jar(): + """No class refers to a package that only an excluded jar provides.""" + dangling = _scan(get_jar_path()) + unexplained = sorted( + (jar, package, example) + for (jar, package), example in dangling.items() + if not _is_allowed(jar, package) + ) + assert not unexplained, ( + "classes in the wheel refer to packages no shipped jar provides, so the " + "code path that uses them fails with NoClassDefFoundError:\n" + + "\n".join( + f" {package.replace('/', '.')} (from {jar}, e.g. {example})" + for jar, package, example in unexplained + ) + + "\nShip the jar that provides it (remove it from scripts/jar_exclusions.txt)," + " or, when the code path cannot be reached from the bindings, add an entry" + " with a reason to ALLOWED_DANGLING in this file." + ) + + +def test_scanner_reports_a_package_whose_jar_is_missing(tmp_path): + """The scan reports a reference whose provider is absent, so a pass means something. + + A directory holding only the jvector jar leaves its dependencies (agrona, + commons-math3, slf4j, ...) unprovided, and the scan must say so. + """ + jar_dir = get_jar_path() + jvector = [n for n in os.listdir(jar_dir) if n.startswith("jvector-")] + assert jvector, "jvector jar missing from the wheel" + shutil.copy(os.path.join(jar_dir, jvector[0]), tmp_path / jvector[0]) + packages = {package for (_, package) in _scan(str(tmp_path))} + assert "org/agrona/collections" in packages + assert not any(p.startswith("io/github/jbellis/jvector") for p in packages) + + +def test_class_reference_parser_reads_a_real_class(): + """The constant-pool parser returns the classes a known class names.""" + jar_dir = get_jar_path() + engine = [n for n in os.listdir(jar_dir) if n.startswith("arcadedb-engine-")][0] + with zipfile.ZipFile(os.path.join(jar_dir, engine)) as archive: + refs = _class_references(archive.read("com/arcadedb/Constants.class")) + assert "com/arcadedb/Constants" in refs + assert "java/lang/String" in refs + + +def test_polyglot_engine_is_off_while_no_graalvm_language_ships(tmp_path): + """The engine is off by default because the wheel carries no language for it. + + GraalJS (js-language, truffle, icu4j, regex) is in jar_exclusions.txt, so a + polyglot engine would start with nothing to register and log "no languages + found" at the first SQL command, and ``DEFINE FUNCTION ... LANGUAGE js`` + cannot work either way. If a language jar is ever shipped, this test stops + asserting either half. + """ + import arcadedb_embedded as arcadedb + import jpype + from arcadedb_embedded.exceptions import ArcadeDBError + + service = ( + "META-INF/services/com.oracle.truffle.api.provider.TruffleLanguageProvider" + ) + jar_dir = get_jar_path() + language_shipped = False + for name in os.listdir(jar_dir): + if name.endswith(".jar"): + with zipfile.ZipFile(os.path.join(jar_dir, name)) as archive: + language_shipped = language_shipped or service in archive.namelist() + + with arcadedb.create_database(str(tmp_path / "poly")) as db: + configuration = jpype.JClass("com.arcadedb.GlobalConfiguration") + enabled = bool(configuration.POLYGLOT_ENGINE_ENABLED.getValueAsBoolean()) + assert language_shipped or not enabled, ( + "the polyglot engine is enabled but no GraalVM language is in the wheel; " + "start_jvm() should default -Darcadedb.polyglotEngineEnabled=false" + ) + if not language_shipped: + with pytest.raises(ArcadeDBError): + db.command( + "sql", + 'DEFINE FUNCTION jsfn.add "return a + b;" PARAMETERS [a,b] LANGUAGE js', + ) + + +# The JDK modules that scripts/jre_module_exclusions.txt keeps out of the bundled +# runtime, with the packages each one provides in the full JDK. A package that is +# in this table and that the running JVM does not provide is "removed on purpose". +# java.datatransfer and java.prefs go with java.desktop, the only module that +# requires them. +REMOVED_MODULES = ( + "java.desktop", + "java.datatransfer", + "java.prefs", + "jdk.jfr", + "java.compiler", +) +REMOVED_MODULE_PACKAGES = ( + "java/awt", # java.desktop, and java/awt/datatransfer of java.datatransfer + "java/beans", + "java/applet", + "java/util/prefs", + "javax/swing", + "javax/imageio", + "javax/sound", + "javax/print", + "javax/accessibility", + "javax/tools", # java.compiler + "javax/lang/model", + "javax/annotation/processing", + "jdk/jfr", + "sun/awt", + "sun/java2d", + "sun/swing", + "com/sun/beans", + "com/sun/java/swing", +) + +# (referencing jar name prefix, package prefix, why the code path is unreachable). +# Same rule as ALLOWED_DANGLING: the entry excuses only that jar and that package. +# Every entry was checked by reading the callers in the shipped jars (2026-10-22): +# no class outside the named one refers to it. +ALLOWED_REMOVED_MODULE_USE = ( + ( + "jts-core-", + "java/awt", + "org.locationtech.jts.awt (AWT shape reader and writer for drawing) and " + "util.TestBuilderProxy. ArcadeDB parses, indexes and tests geometries and " + "never refers to the awt package, and no other JTS class does.", + ), + ( + "jackson-databind-", + "java/beans", + "ext.Java7SupportImpl (@ConstructorProperties and @Transient). Jackson " + "loads it in a try block and works without that annotation support; " + "test_jackson_and_yaml_work_without_java_beans runs it on this runtime.", + ), + ( + "snakeyaml-", + "java/beans", + "introspector.MethodProperty, the bean mapper of Yaml, Constructor and " + "Representer. Nothing in the wheel creates a Yaml object: " + "jackson-dataformat-yaml uses only the parser and the emitter.", + ), + ( + "commons-lang3-", + "java/beans", + "concurrent.AbstractCircuitBreaker (PropertyChangeSupport), which nothing " + "in the wheel extends.", + ), + ( + "commons-math3-", + "java/awt", + "geometry.euclidean.twod.Line (AffineTransform), a 2-D geometry class. " + "jvector refers to other commons-math3 classes only.", + ), + ( + "micrometer-core-", + "java/beans", + "binder.logging.Log4j2Metrics, bound only by an application that registers " + "it; the wheel logs through java.util.logging and ships no Log4j2.", + ), + ( + "jakarta.xml.bind-api-", + "java/beans", + "javax.xml.bind.JAXB, a convenience class. swagger-core refers to the " + "javax.xml.bind.annotation annotations only, which need no JDK module.", + ), + ( + "jakarta.activation-api-", + "java/awt/datatransfer", + "javax.activation.DataHandler and its content handlers (data flavors for " + "JAXB attachments). No class outside the jar refers to javax.activation.", + ), + ( + "arcadedb-engine-", + "jdk/jfr", + "utility.JVMGCHistogramReporter (Flight Recorder allocation stream). No " + "class, command or setting calls it; the JFR events of the JVM itself " + "need no Java module.", + ), + ( + "agrona-", + "javax/tools", + "org.agrona.generation, agrona's code generator that compiles source at " + "run time. Neither jvector nor the engine calls it.", + ), +) + + +def _runtime_packages(): + """The packages of every module in the running JVM's boot layer, as ``a/b/c``.""" + import jpype + from arcadedb_embedded.jvm import start_jvm + + start_jvm() + modules = jpype.JClass("java.lang.ModuleLayer").boot().modules() + return { + str(package).replace(".", "/") + for module in modules + for package in module.getPackages() + } + + +def _removed_module_references(jar_dir, runtime_packages): + """Return ``{(jar, package): example class}`` for packages of removed modules. + + Unlike ``_scan`` this reads the multi-release classes under + ``META-INF/versions/`` for ``n`` up to the runtime's Java 25, as jdeps + does with ``--multi-release 25``, because the JVM loads them. + """ + found = {} + for jar in sorted(n for n in os.listdir(jar_dir) if n.endswith(".jar")): + with zipfile.ZipFile(os.path.join(jar_dir, jar)) as archive: + for name in archive.namelist(): + if not name.endswith(".class"): + continue + if name.startswith("META-INF/versions/"): + version, _, rest = name[len("META-INF/versions/") :].partition("/") + if not version.isdigit() or int(version) > 25: + continue + class_file = rest + elif name.startswith("META-INF/"): + continue + else: + class_file = name + if class_file == "module-info.class": + continue + for ref in _class_references(archive.read(name)): + package = _package_of(ref) + if package in runtime_packages: + continue + if any( + package == removed or package.startswith(removed + "/") + for removed in REMOVED_MODULE_PACKAGES + ): + found.setdefault((jar, package), class_file) + return found + + +def _removed_module_use_is_allowed(jar, package): + return any( + jar.startswith(jar_prefix) and package.startswith(package_prefix) + for jar_prefix, package_prefix, _ in ALLOWED_REMOVED_MODULE_USE + ) + + +def test_bundled_runtime_lacks_the_excluded_modules(): + """The runtime really is the trimmed one, so the closure test below means something.""" + import jpype + from arcadedb_embedded.jvm import start_jvm + + start_jvm() + layer = jpype.JClass("java.lang.ModuleLayer").boot() + present = [m for m in REMOVED_MODULES if layer.findModule(m).isPresent()] + assert not present, ( + f"the bundled JRE contains {present}, which scripts/jre_module_exclusions.txt " + "keeps out; the wheel was built without that list" + ) + + +def test_no_shipped_class_needs_a_module_the_runtime_lacks(): + """Every class that names a removed module's package is on the allow list.""" + found = _removed_module_references(get_jar_path(), _runtime_packages()) + unexplained = sorted( + (jar, package, example) + for (jar, package), example in found.items() + if not _removed_module_use_is_allowed(jar, package) + ) + assert not unexplained, ( + "classes in the wheel refer to a package of a JDK module that the bundled " + "JRE lacks (scripts/jre_module_exclusions.txt), so the code path that uses " + "them fails with NoClassDefFoundError:\n" + + "\n".join( + f" {package.replace('/', '.')} (from {jar}, e.g. {example})" + for jar, package, example in unexplained + ) + + "\nKeep the module (remove it from jre_module_exclusions.txt), or, when the" + " code path cannot be reached from the bindings, add an entry with a reason" + " to ALLOWED_REMOVED_MODULE_USE in this file." + ) + + +def test_removed_module_scan_sees_the_known_references(): + """With no runtime packages the scan finds the references, so a pass means something.""" + found = _removed_module_references(get_jar_path(), set()) + seen = {(jar.split("-")[0], package) for jar, package in found} + assert ("jackson", "java/beans") in seen + assert ("jts", "java/awt/geom") in seen + assert ("arcadedb", "jdk/jfr/consumer") in seen + assert ("agrona", "javax/tools") in seen + + +def test_allowed_removed_module_entries_still_match_a_reference(): + """An entry that no class needs any more is dead text: remove it.""" + found = _removed_module_references(get_jar_path(), set()) + stale = [ + (jar_prefix, package_prefix) + for jar_prefix, package_prefix, _ in ALLOWED_REMOVED_MODULE_USE + if not any( + jar.startswith(jar_prefix) and package.startswith(package_prefix) + for jar, package in found + ) + ] + assert not stale, f"entries of ALLOWED_REMOVED_MODULE_USE match nothing: {stale}" + + +def test_jackson_and_yaml_work_without_java_beans(tmp_path): + """Jackson and the YAML mapper run on this runtime, which has no java.beans. + + The allow list says jackson-databind's java.beans use is optional and that the + YAML mapper does not touch snakeyaml's bean introspector. Run both. + """ + import jpype + from arcadedb_embedded.jvm import start_jvm + + start_jvm() + mapper = jpype.JClass("com.fasterxml.jackson.databind.ObjectMapper")() + tree = mapper.readTree('{"a":[1,2,{"b":"c"}],"d":null}') + assert str(mapper.writeValueAsString(tree)) == '{"a":[1,2,{"b":"c"}],"d":null}' + hash_map = jpype.JClass("java.util.LinkedHashMap")() + hash_map.put("name", "arcade") + hash_map.put("n", jpype.JClass("java.lang.Integer")(3)) + assert "arcade" in str(mapper.writeValueAsString(hash_map)) + + yaml = jpype.JClass("com.fasterxml.jackson.dataformat.yaml.YAMLMapper")() + text = str(yaml.writeValueAsString(hash_map)) + assert "name: " in text and "arcade" in text + back = yaml.readTree(text) + assert str(back.get("name").asText()) == "arcade" diff --git a/tests/test_server_packaging.py b/tests/test_server_packaging.py index d1e0dab..4a25bdf 100644 --- a/tests/test_server_packaging.py +++ b/tests/test_server_packaging.py @@ -123,3 +123,102 @@ def test_server_starts_and_serves_http(temp_server_root): ) assert r.status_code == 200, f"server returned {r.status_code}" assert "version" in r.json() + + +def _varint(n): + out = bytearray() + while True: + low = n & 0x7F + n >>= 7 + if n: + out.append(low | 0x80) + else: + out.append(low) + return bytes(out) + + +def _field(number, payload): + return _varint(number << 3 | 2) + _varint(len(payload)) + payload + + +def _snappy_prometheus_write(): + """A Prometheus remote-write body: one sample, protobuf, snappy literal block.""" + import struct + + label = _field(1, _field(1, b"__name__") + _field(2, b"cpu")) + sample = _field(2, b"\x09" + struct.pack(" Date: Sat, 24 Oct 2026 10:39:49 +0900 Subject: [PATCH 2/2] Move the polyglot-default test out; README package sizes measured on the built wheel The polyglot test checks the -Darcadedb.polyglotEngineEnabled=false default in jvm.py, which comes with the runtime-fixes port. The README numbers are from the 26.11.1.dev0 linux/amd64 wheel built from this tree. Refs #19 Co-Authored-By: Claude Opus 5.5 --- README.md | 14 +++++++------- tests/test_jar_closure.py | 38 -------------------------------------- 2 files changed, 7 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index 0bec0c4..732b3ba 100644 --- a/README.md +++ b/README.md @@ -56,11 +56,11 @@ with arcadedb.create_database("./mydb") as db: ## ✨ Features -- ☕ **No Java Installation Required**: Bundled JRE (~63 MiB uncompressed) +- ☕ **No Java Installation Required**: Bundled JRE (~44 MiB uncompressed) - 🌍 **4 Platforms Supported**: Linux (x86_64, ARM64), macOS (ARM64), Windows (x86_64) - 🚀 **Embedded Mode**: Direct database access in Python process (no network) - 🌐 **Server Mode**: Optional in-process HTTP server with the Studio web UI: it costs ~8MB of wheel and nothing at runtime until you call `create_server()` -- 📦 **Self-contained**: All dependencies bundled (~69 MiB current Linux wheel) +- 📦 **Self-contained**: All dependencies bundled (~64 MiB current Linux wheel) - 🔄 **Multi-model**: Graph, Document, Key/Value, Vector, Time Series - 🔍 **Multiple query languages**: SQL and OpenCypher - ⚡ **High performance**: Direct JVM integration via JPype @@ -74,12 +74,12 @@ with arcadedb.create_database("./mydb") as db: The `arcadedb-embedded` package is platform-specific and self-contained: -**Package Contents (the 26.10.1 Linux x86_64 wheel; varies by platform and version):** +**Package Contents (the 26.11.1.dev0 Linux x86_64 wheel; varies by platform and version):** -- **Wheel size (compressed)**: ~69 MiB -- **ArcadeDB JARs (uncompressed)**: ~34 MiB across 64 JARs -- **Bundled JRE (uncompressed)**: ~63 MiB (platform-specific Java 25 runtime via jlink) -- **Installed package size**: ~97 MiB +- **Wheel size (compressed)**: ~64 MiB +- **ArcadeDB JARs (uncompressed)**: ~42 MiB across 72 JARs +- **Bundled JRE (uncompressed)**: ~44 MiB (platform-specific Java 25 runtime via jlink) +- **Installed package size**: ~86 MiB The compressed wheel size is measured from `dist/*.whl`, and the installed package size is measured from the extracted `site-packages/arcadedb_embedded/` directory. diff --git a/tests/test_jar_closure.py b/tests/test_jar_closure.py index 16f8f44..20d627d 100644 --- a/tests/test_jar_closure.py +++ b/tests/test_jar_closure.py @@ -250,44 +250,6 @@ def test_class_reference_parser_reads_a_real_class(): assert "java/lang/String" in refs -def test_polyglot_engine_is_off_while_no_graalvm_language_ships(tmp_path): - """The engine is off by default because the wheel carries no language for it. - - GraalJS (js-language, truffle, icu4j, regex) is in jar_exclusions.txt, so a - polyglot engine would start with nothing to register and log "no languages - found" at the first SQL command, and ``DEFINE FUNCTION ... LANGUAGE js`` - cannot work either way. If a language jar is ever shipped, this test stops - asserting either half. - """ - import arcadedb_embedded as arcadedb - import jpype - from arcadedb_embedded.exceptions import ArcadeDBError - - service = ( - "META-INF/services/com.oracle.truffle.api.provider.TruffleLanguageProvider" - ) - jar_dir = get_jar_path() - language_shipped = False - for name in os.listdir(jar_dir): - if name.endswith(".jar"): - with zipfile.ZipFile(os.path.join(jar_dir, name)) as archive: - language_shipped = language_shipped or service in archive.namelist() - - with arcadedb.create_database(str(tmp_path / "poly")) as db: - configuration = jpype.JClass("com.arcadedb.GlobalConfiguration") - enabled = bool(configuration.POLYGLOT_ENGINE_ENABLED.getValueAsBoolean()) - assert language_shipped or not enabled, ( - "the polyglot engine is enabled but no GraalVM language is in the wheel; " - "start_jvm() should default -Darcadedb.polyglotEngineEnabled=false" - ) - if not language_shipped: - with pytest.raises(ArcadeDBError): - db.command( - "sql", - 'DEFINE FUNCTION jsfn.add "return a + b;" PARAMETERS [a,b] LANGUAGE js', - ) - - # The JDK modules that scripts/jre_module_exclusions.txt keeps out of the bundled # runtime, with the packages each one provides in the full JDK. A package that is # in this table and that the running JVM does not provide is "removed on purpose".